IMPALA-1587: Extending caching directives for multiple replicas

This patch adds the possibility to specify the number of replicas that
should be cached in main memory. This can be useful in high QPS
scenarios as the majority of the load is no longer the single cached
replica, but a set of cached replicas. While the cache replication
factor can be larger than the block replication factor on disk, the
difference will be ignored by HDFS until more replicas become
available.

This extends the current syntax for specifying the cache pool in the
following way:

   cached in 'poolName'

is extended with the optional replication factor

   cached in 'poolName' with replication = XX

By default, the cache replication factor is set to 1. As this value is
not yet configurable in HDFS it's defined as a constant in the JniCatalog
thrift specification. If a partitioned table is cached, all its child
partitions inherit this cache replication factor. If child partitions
have a custom cache replication factor, changing the cache replication
factor on the partitioned table afterwards will overwrite this custom
value. If a new partition is added to the table, it will again inherit
the cache replication factor of the parent independent of the cache pool
that is used to cache the partition.

To review changes and status of the replication factor for tables and
partitions the replication factor is part of output of the "show
partitions" command.

Change-Id: I2aee63258d6da14fb5ce68574c6b070cf948fb4d
Reviewed-on: http://gerrit.sjc.cloudera.com:8080/5533
Tested-by: jenkins
Reviewed-by: Martin Grund <mgrund@cloudera.com>
This commit is contained in:
Martin Grund
2014-12-02 09:20:27 -08:00
parent ba081abac9
commit cee1e84c1e
21 changed files with 3439 additions and 3023 deletions

View File

@@ -20,6 +20,9 @@ include "Types.thrift"
include "Status.thrift"
include "TCLIService.thrift"
// This is a short value due to the HDFS API limits
const i16 HDFS_DEFAULT_CACHE_REPLICATION_FACTOR = 1
// Structs used to execute DDL operations using the JniCatalog.
enum TDdlType {
@@ -140,6 +143,11 @@ struct THdfsCachingOp {
// Set only if set_cached=true. Provides the name of the pool to use when caching.
2: optional string cache_pool_name
// The optional cache replication factor to use. If the replication factor is not
// specified it's either inherited from the table if the underlying object is a
// partition or is set to our default HDFS cache replication factor.
3: optional i16 replication
}
// Parameters for ALTER TABLE rename commands

View File

@@ -245,11 +245,11 @@ terminal
KW_OFFSET, KW_ON, KW_OR, KW_ORDER, KW_OUTER, KW_OVER, KW_OVERWRITE, KW_PARQUET,
KW_PARQUETFILE, KW_PARTITION, KW_PARTITIONED, KW_PARTITIONS, KW_PRECEDING,
KW_PREPARE_FN, KW_PRODUCED, KW_RANGE, KW_RCFILE, KW_REFRESH, KW_REGEXP, KW_RENAME,
KW_REPLACE, KW_RETURNS, KW_REVOKE, KW_RIGHT, KW_RLIKE, KW_ROLE, KW_ROLES, KW_ROW,
KW_ROWS, KW_SCHEMA, KW_SCHEMAS, KW_SELECT, KW_SEMI, KW_SEQUENCEFILE, KW_SERDEPROPERTIES,
KW_SERIALIZE_FN, KW_SET, KW_SHOW, KW_SMALLINT, KW_STORED, KW_STRAIGHT_JOIN,
KW_STRING, KW_STRUCT, KW_SYMBOL, KW_TABLE, KW_TABLES, KW_TBLPROPERTIES, KW_TERMINATED,
KW_TEXTFILE, KW_THEN,
KW_REPLACE, KW_REPLICATION, KW_RETURNS, KW_REVOKE, KW_RIGHT, KW_RLIKE, KW_ROLE,
KW_ROLES, KW_ROW, KW_ROWS, KW_SCHEMA, KW_SCHEMAS, KW_SELECT, KW_SEMI, KW_SEQUENCEFILE,
KW_SERDEPROPERTIES, KW_SERIALIZE_FN, KW_SET, KW_SHOW, KW_SMALLINT, KW_STORED,
KW_STRAIGHT_JOIN, KW_STRING, KW_STRUCT, KW_SYMBOL, KW_TABLE, KW_TABLES,
KW_TBLPROPERTIES, KW_TERMINATED, KW_TEXTFILE, KW_THEN,
KW_TIMESTAMP, KW_TINYINT, KW_STATS, KW_TO, KW_TRUE, KW_UNBOUNDED, KW_UNCACHED,
KW_UNION, KW_UPDATE_FN, KW_USE, KW_USING,
KW_VALUES, KW_VARCHAR, KW_VIEW, KW_WHEN, KW_WHERE, KW_WITH;
@@ -375,6 +375,7 @@ nonterminal ArrayList<ColumnDesc> partition_column_defs, view_column_defs;
nonterminal ArrayList<StructField> struct_field_def_list;
// Options for DDL commands - CREATE/DROP/ALTER
nonterminal HdfsCachingOp cache_op_val;
nonterminal BigDecimal opt_cache_op_replication;
nonterminal String comment_val;
nonterminal Boolean external_val;
nonterminal String opt_init_string_val;
@@ -939,14 +940,21 @@ create_uda_stmt ::=
;
cache_op_val ::=
KW_CACHED KW_IN STRING_LITERAL:pool_name
{: RESULT = new HdfsCachingOp(pool_name); :}
KW_CACHED KW_IN STRING_LITERAL:pool_name opt_cache_op_replication:replication
{: RESULT = new HdfsCachingOp(pool_name, replication); :}
| KW_UNCACHED
{: RESULT = new HdfsCachingOp(); :}
| /* empty */
{: RESULT = null; :}
;
opt_cache_op_replication ::=
KW_WITH KW_REPLICATION EQUAL INTEGER_LITERAL:replication
{: RESULT = replication; :}
| /* empty */
{: RESULT = null; :}
;
comment_val ::=
KW_COMMENT STRING_LITERAL:comment
{: RESULT = comment; :}

View File

@@ -14,7 +14,8 @@
package com.cloudera.impala.analysis;
import com.cloudera.impala.catalog.AuthorizationException;
import java.math.BigDecimal;
import com.cloudera.impala.catalog.HdfsCachePool;
import com.cloudera.impala.common.AnalysisException;
import com.cloudera.impala.thrift.THdfsCachingOp;
@@ -26,20 +27,24 @@ import com.google.common.base.Preconditions;
*/
public class HdfsCachingOp implements ParseNode {
private final THdfsCachingOp cacheOp_;
private final BigDecimal parsedReplication_;
/**
* Creates an HdfsCachingOp that specifies the target should be uncached
*/
public HdfsCachingOp() {
cacheOp_ = new THdfsCachingOp(false);
parsedReplication_ = null;
}
/**
* Creates an HdfsCachingOp that specifies the target should be cached in cachePoolName
* with an optional replication factor
*/
public HdfsCachingOp(String cachePoolName) {
public HdfsCachingOp(String cachePoolName, BigDecimal replication) {
cacheOp_ = new THdfsCachingOp(true);
cacheOp_.setCache_pool_name(cachePoolName);
parsedReplication_ = replication;
}
@Override
@@ -56,17 +61,29 @@ public class HdfsCachingOp implements ParseNode {
throw new AnalysisException(
"The specified cache pool does not exist: " + poolName);
}
if (parsedReplication_ != null && (parsedReplication_.longValue() <= 0 ||
parsedReplication_.longValue() > Short.MAX_VALUE)) {
throw new AnalysisException(
"Cache replication factor must be between 0 and Short.MAX_VALUE");
}
if (parsedReplication_ != null) {
cacheOp_.setReplication(parsedReplication_.shortValue());
}
}
}
@Override
public String toSql() {
return shouldCache() ? "CACHED IN '" + getCachePoolName() + "'" : "UNCACHED";
return !shouldCache() ? "UNCACHED" : "CACHED IN '" + getCachePoolName() + "' WITH " +
"REPLICATION = " + parsedReplication_.longValue();
}
public THdfsCachingOp toThrift() { return cacheOp_; }
public boolean shouldCache() { return cacheOp_.isSet_cached(); }
public String getCachePoolName() {
return shouldCache() ? cacheOp_.getCache_pool_name() : null;
}

View File

@@ -24,14 +24,12 @@ import org.apache.commons.lang.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudera.impala.analysis.ToSqlUtils;
import com.cloudera.impala.analysis.Expr;
import com.cloudera.impala.analysis.LiteralExpr;
import com.cloudera.impala.analysis.NullLiteral;
import com.cloudera.impala.analysis.PartitionKeyValue;
import com.cloudera.impala.catalog.PartitionStatsUtil;
import com.cloudera.impala.analysis.ToSqlUtils;
import com.cloudera.impala.common.ImpalaException;
import com.cloudera.impala.common.JniUtil;
import com.cloudera.impala.thrift.ImpalaInternalServiceConstants;
import com.cloudera.impala.thrift.TAccessLevel;
import com.cloudera.impala.thrift.TExpr;
@@ -41,15 +39,15 @@ import com.cloudera.impala.thrift.THdfsFileBlock;
import com.cloudera.impala.thrift.THdfsFileDesc;
import com.cloudera.impala.thrift.THdfsPartition;
import com.cloudera.impala.thrift.TNetworkAddress;
import com.cloudera.impala.thrift.TTableStats;
import com.cloudera.impala.thrift.TPartitionStats;
import com.cloudera.impala.thrift.TTableStats;
import com.cloudera.impala.util.HdfsCachingUtil;
import com.google.common.base.Joiner;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.base.Joiner;
/**
* Query-relevant information for one table partition. Partitions are comparable
@@ -253,6 +251,8 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
* It's easy to add per-file metadata to FileDescriptor if this changes.
*/
private final HdfsStorageDescriptor fileFormatDescriptor_;
// Note: this field is only set in the catalog server
private final org.apache.hadoop.hive.metastore.api.Partition msPartition_;
private final List<FileDescriptor> fileDescriptors_;
private final String location_;
@@ -275,6 +275,8 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
* Returns the metastore.api.Partition object this HdfsPartition represents. Returns
* null if this is the default partition, or if this belongs to a unpartitioned
* table.
*
* Note: The return value of this method has no meaning, when called by an impalad
*/
public org.apache.hadoop.hive.metastore.api.Partition getMetaStorePartition() {
return msPartition_;
@@ -393,6 +395,8 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
return hmsParameters_.get(key);
}
public Map<String, String> getHmsParameters() { return hmsParameters_; }
/**
* Marks this partition's metadata as "dirty" indicating that changes have been
* made and this partition's metadata should not be reused during the next
@@ -426,7 +430,7 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
id_ = id;
accessLevel_ = accessLevel;
if (msPartition != null && msPartition.getParameters() != null) {
isMarkedCached_ = HdfsCachingUtil.getCacheDirIdFromParams(
isMarkedCached_ = HdfsCachingUtil.getCacheDirectiveId(
msPartition.getParameters()) != null;
hmsParameters_ = msPartition.getParameters();
} else {

View File

@@ -34,7 +34,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStorageLocation;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -603,9 +602,10 @@ public class HdfsTable extends Table {
// Scans don't refer to this because by definition all partitions they refer to
// exist.
addDefaultPartition(msTbl.getSd());
Long cacheDirectiveId =
HdfsCachingUtil.getCacheDirIdFromParams(msTbl.getParameters());
isMarkedCached_ = cacheDirectiveId != null;
// We silently ignore cache directives that no longer exist in HDFS
isMarkedCached_ =
HdfsCachingUtil.getCacheDirectiveId(msTbl.getParameters()) != null;
if (msTbl.getPartitionKeysSize() == 0) {
Preconditions.checkArgument(msPartitions == null || msPartitions.isEmpty());
@@ -733,7 +733,7 @@ public class HdfsTable extends Table {
List<LiteralExpr> keyValues = Lists.newArrayList();
if (msPartition != null) {
isMarkedCached =
HdfsCachingUtil.getCacheDirIdFromParams(msPartition.getParameters()) != null;
HdfsCachingUtil.getCacheDirectiveId(msPartition.getParameters()) != null;
// Load key values
for (String partitionKey: msPartition.getValues()) {
Type type = getColumns().get(keyValues.size()).getType();
@@ -1220,7 +1220,7 @@ public class HdfsTable extends Table {
partitions_.add(hdfsPart);
}
avroSchema_ = hdfsTable.isSetAvroSchema() ? hdfsTable.getAvroSchema() : null;
isMarkedCached_ = HdfsCachingUtil.getCacheDirIdFromParams(
isMarkedCached_ = HdfsCachingUtil.getCacheDirectiveId(
getMetaStoreTable().getParameters()) != null;
populatePartitionMd();
}
@@ -1336,6 +1336,7 @@ public class HdfsTable extends Table {
resultSchema.addToColumns(new TColumn("#Files", Type.BIGINT.toThrift()));
resultSchema.addToColumns(new TColumn("Size", Type.STRING.toThrift()));
resultSchema.addToColumns(new TColumn("Bytes Cached", Type.STRING.toThrift()));
resultSchema.addToColumns(new TColumn("Cache Replication", Type.STRING.toThrift()));
resultSchema.addToColumns(new TColumn("Format", Type.STRING.toThrift()));
resultSchema.addToColumns(new TColumn("Incremental stats", Type.STRING.toThrift()));
@@ -1361,6 +1362,7 @@ public class HdfsTable extends Table {
// Helps to differentiate partitions that have 0B cached versus partitions
// that are not marked as cached.
rowBuilder.add("NOT CACHED");
rowBuilder.add("NOT CACHED");
} else {
// Calculate the number the number of bytes that are cached.
long cachedBytes = 0L;
@@ -1373,6 +1375,14 @@ public class HdfsTable extends Table {
}
totalCachedBytes += cachedBytes;
rowBuilder.addBytes(cachedBytes);
// Extract cache replication factor from the parameters of the table
// if the table is not partitioned or directly from the partition.
Short rep = HdfsCachingUtil.getCachedCacheReplication(
numClusteringCols_ == 0 ?
p.getTable().getMetaStoreTable().getParameters() :
p.getHmsParameters());
rowBuilder.add(rep.toString());
}
rowBuilder.add(p.getInputFormatDescriptor().getFileFormat().toString());
@@ -1391,7 +1401,7 @@ public class HdfsTable extends Table {
// Total num rows, files, and bytes (leave format empty).
rowBuilder.add(numRows_).add(numHdfsFiles_).addBytes(totalHdfsBytes_)
.addBytes(totalCachedBytes).add("").add("");
.addBytes(totalCachedBytes).add("").add("").add("");
result.addToRows(rowBuilder.get());
}
return result;

View File

@@ -204,7 +204,7 @@ public class TableLoadingMgr {
pendingTableCacheDirs_.put(tblName, cacheDirIds);
refreshThreadWork_.add(tblName);
} else {
cacheDirIds.addAll(cacheDirIds);
existingCacheReqIds.addAll(cacheDirIds);
}
}
}

View File

@@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
@@ -78,6 +77,7 @@ import com.cloudera.impala.common.ImpalaRuntimeException;
import com.cloudera.impala.common.InternalException;
import com.cloudera.impala.common.Pair;
import com.cloudera.impala.thrift.ImpalaInternalServiceConstants;
import com.cloudera.impala.thrift.JniCatalogConstants;
import com.cloudera.impala.thrift.TAlterTableAddPartitionParams;
import com.cloudera.impala.thrift.TAlterTableAddReplaceColsParams;
import com.cloudera.impala.thrift.TAlterTableChangeColParams;
@@ -117,8 +117,8 @@ import com.cloudera.impala.thrift.TGrantRevokeRoleParams;
import com.cloudera.impala.thrift.THdfsCachingOp;
import com.cloudera.impala.thrift.THdfsFileFormat;
import com.cloudera.impala.thrift.TPartitionKeyValue;
import com.cloudera.impala.thrift.TPrivilege;
import com.cloudera.impala.thrift.TPartitionStats;
import com.cloudera.impala.thrift.TPrivilege;
import com.cloudera.impala.thrift.TResetMetadataRequest;
import com.cloudera.impala.thrift.TResetMetadataResponse;
import com.cloudera.impala.thrift.TResultRow;
@@ -132,7 +132,6 @@ import com.cloudera.impala.thrift.TTableStats;
import com.cloudera.impala.thrift.TUpdateCatalogRequest;
import com.cloudera.impala.thrift.TUpdateCatalogResponse;
import com.cloudera.impala.util.HdfsCachingUtil;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -1228,8 +1227,10 @@ public class CatalogOpExecutor {
// Submit the cache request and update the table metadata.
if (cacheOp != null && cacheOp.isSet_cached()) {
short replication = cacheOp.isSetReplication() ? cacheOp.getReplication() :
JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR;
long id = HdfsCachingUtil.submitCacheTblDirective(newTable,
cacheOp.getCache_pool_name());
cacheOp.getCache_pool_name(), replication);
catalog_.watchCacheDirs(Lists.<Long>newArrayList(id),
new TTableName(newTable.getDbName(), newTable.getTableName()));
applyAlterTable(newTable);
@@ -1337,13 +1338,16 @@ public class CatalogOpExecutor {
" and ifNotExists is true.", Joiner.on(", ").join(partitionSpec)));
return null;
}
Table result = null;
List<Long> cacheIds = null;
synchronized (metastoreDdlLock_) {
org.apache.hadoop.hive.metastore.api.Table msTbl = getMetaStoreTable(tableName);
partition.setDbName(tableName.getDb());
partition.setTableName(tableName.getTbl());
Long parentTblCacheDirId =
HdfsCachingUtil.getCacheDirIdFromParams(msTbl.getParameters());
HdfsCachingUtil.getCacheDirectiveId(msTbl.getParameters());
List<String> values = Lists.newArrayList();
// Need to add in the values in the same order they are defined in the table.
@@ -1363,21 +1367,33 @@ public class CatalogOpExecutor {
// Add the new partition.
partition = msClient.getHiveClient().add_partition(partition);
String cachePoolName = null;
Short replication = null;
if (cacheOp == null && parentTblCacheDirId != null) {
// The user didn't specify an explicit caching operation, inherit the value
// from the parent table.
cachePoolName = HdfsCachingUtil.getCachePool(parentTblCacheDirId);
Preconditions.checkNotNull(cachePoolName);
replication = HdfsCachingUtil.getCacheReplication(parentTblCacheDirId);
Preconditions.checkNotNull(replication);
} else if (cacheOp != null && cacheOp.isSet_cached()) {
// The explicitly stated that this partition should be cached.
// The user explicitly stated that this partition should be cached.
cachePoolName = cacheOp.getCache_pool_name();
}
// When the new partition should be cached and and no replication factor
// was specified, inherit the replication factor from the parent table if
// it is cached. If the parent is not cached and no replication factor is
// explicitly set, use the default value.
if (!cacheOp.isSetReplication() && parentTblCacheDirId != null) {
replication = HdfsCachingUtil.getCacheReplication(parentTblCacheDirId);
} else {
replication = HdfsCachingUtil.getReplicationOrDefault(cacheOp);
}
}
// If cache pool name is not null, it indicates this partition should be cached.
if (cachePoolName != null) {
long id = HdfsCachingUtil.submitCachePartitionDirective(partition,
cachePoolName);
catalog_.watchCacheDirs(
Lists.<Long>newArrayList(id), tableName.toThrift());
cachePoolName, replication);
cacheIds = Lists.<Long>newArrayList(id);
// Update the partition metadata to include the cache directive id.
msClient.getHiveClient().alter_partition(partition.getDbName(),
partition.getTableName(), partition);
@@ -1399,7 +1415,9 @@ public class CatalogOpExecutor {
}
// Create and add the HdfsPartition. Return the table object with an updated catalog
// version.
return addHdfsPartition(tableName, partition);
result = addHdfsPartition(tableName, partition);
if (cacheIds != null) catalog_.watchCacheDirs(cacheIds, tableName.toThrift());
return result;
}
/**
@@ -1666,24 +1684,24 @@ public class CatalogOpExecutor {
}
HdfsTable hdfsTable = (HdfsTable) table;
org.apache.hadoop.hive.metastore.api.Table msTbl = table.getMetaStoreTable();
Long cacheDirId = HdfsCachingUtil.getCacheDirIdFromParams(msTbl.getParameters());
Long cacheDirId = HdfsCachingUtil.getCacheDirectiveId(msTbl.getParameters());
if (cacheOp.isSet_cached()) {
// List of cache directive IDs that were submitted as part of this
// ALTER TABLE operation.
List<Long> cacheDirIds = Lists.newArrayList();
short cacheReplication = HdfsCachingUtil.getReplicationOrDefault(cacheOp);
// If the table was not previously cached (cacheDirId == null) we issue a new
// directive for this table. If the table was already cached, we validate
// the pool name and update the cache replication factor if necessary
if (cacheDirId == null) {
// Table was not already cached.
cacheDirIds.add(HdfsCachingUtil.submitCacheTblDirective(msTbl,
cacheOp.getCache_pool_name()));
cacheOp.getCache_pool_name(), cacheReplication));
} else {
// Table is already cached, verify the pool name doesn't conflict.
String pool = HdfsCachingUtil.getCachePool(cacheDirId);
if (!cacheOp.getCache_pool_name().equals(pool)) {
throw new ImpalaRuntimeException(String.format("Cannot cache table in " +
"pool '%s' because it is already cached in pool '%s'. To change the " +
"pool for this table, first uncache using: ALTER TABLE %s.%s SET UNCACHED",
cacheOp.getCache_pool_name(), pool, msTbl.getDbName(),
msTbl.getTableName()));
// Check if the cache directive needs to be changed
if (HdfsCachingUtil.isUpdateOp(cacheOp, msTbl.getParameters())) {
HdfsCachingUtil.validateCachePool(cacheOp, cacheDirId, tableName);
cacheDirIds.add(HdfsCachingUtil.modifyCacheDirective(cacheDirId, msTbl,
cacheOp.getCache_pool_name(), cacheReplication));
}
}
@@ -1699,12 +1717,31 @@ public class CatalogOpExecutor {
org.apache.hadoop.hive.metastore.api.Partition msPart =
partition.getMetaStorePartition();
Preconditions.checkNotNull(msPart);
if (!partition.isMarkedCached()) {
// Only issue cache directives if the data is uncached or the cache directive
// needs to be updated
if (!partition.isMarkedCached() ||
HdfsCachingUtil.isUpdateOp(cacheOp, msPart.getParameters())) {
try {
cacheDirIds.add(HdfsCachingUtil.submitCachePartitionDirective(
msPart, cacheOp.getCache_pool_name()));
// If the partition was already cached, update the directive otherwise
// issue new cache directive
if (!partition.isMarkedCached()) {
cacheDirIds.add(HdfsCachingUtil.submitCachePartitionDirective(
msPart, cacheOp.getCache_pool_name(), cacheReplication));
} else {
Long directiveId = HdfsCachingUtil.getCacheDirectiveId(msPart.getParameters());
cacheDirIds.add(HdfsCachingUtil.modifyCacheDirective(
directiveId, msPart, cacheOp.getCache_pool_name(), cacheReplication));
}
} catch (ImpalaRuntimeException e) {
LOG.error("Unable to cache partition: " + partition.getPartitionName(), e);
if (partition.isMarkedCached()) {
LOG.error("Unable to modify cache partition: " +
partition.getPartitionName(), e);
} else {
LOG.error("Unable to cache partition: " +
partition.getPartitionName(), e);
}
}
// Update the partition metadata.
@@ -1771,24 +1808,30 @@ public class CatalogOpExecutor {
partition.getMetaStorePartition();
Preconditions.checkNotNull(msPartition);
if (cacheOp.isSet_cached()) {
if (partition.isMarkedCached()) {
Long cacheReq = HdfsCachingUtil.getCacheDirIdFromParams(
partition.getMetaStorePartition().getParameters());
String pool = HdfsCachingUtil.getCachePool(cacheReq);
if (!cacheOp.getCache_pool_name().equals(pool)) {
throw new ImpalaRuntimeException(String.format("Cannot cache partition in " +
"pool '%s' because it is already cached in '%s'. To change the cache " +
"pool for this partition, first uncache using: ALTER TABLE %s.%s " +
"PARTITION(%s) SET UNCACHED", cacheOp.getCache_pool_name(), pool,
tableName.getDb(), tableName,
partition.getPartitionName().replaceAll("/", ", ")));
// The directive is null if the partition is not cached
Long directiveId = HdfsCachingUtil.getCacheDirectiveId(
msPartition.getParameters());
short replication = HdfsCachingUtil.getReplicationOrDefault(cacheOp);
List<Long> cacheDirs = Lists.newArrayList();
if (directiveId == null) {
cacheDirs.add(HdfsCachingUtil.submitCachePartitionDirective(msPartition,
cacheOp.getCache_pool_name(), replication));
} else {
if (HdfsCachingUtil.isUpdateOp(cacheOp, msPartition.getParameters())) {
HdfsCachingUtil.validateCachePool(cacheOp, directiveId, tableName, partition);
cacheDirs.add(HdfsCachingUtil.modifyCacheDirective(directiveId, msPartition,
cacheOp.getCache_pool_name(), replication));
}
// Partition is already cached. Nothing to do.
return;
}
long id = HdfsCachingUtil.submitCachePartitionDirective(msPartition,
cacheOp.getCache_pool_name());
catalog_.watchCacheDirs(Lists.<Long>newArrayList(id), tableName.toThrift());
// Once the cache directives are sbumitted, observe the status of the caching
// until no more progress is made -- either fully cached or out of cache memory
if (!cacheDirs.isEmpty()) {
catalog_.watchCacheDirs(cacheDirs, tableName.toThrift());
}
} else {
// Partition is not cached, just return.
if (!partition.isMarkedCached()) return;
@@ -2137,6 +2180,7 @@ public class CatalogOpExecutor {
private final TableName tblName_;
private final String partName_;
private final String cachePoolName_;
private final Short replication_;
private final AtomicBoolean partitionCreated_;
private final AtomicInteger numPartitions_;
private final SettableFuture<Void> allFinished_;
@@ -2155,7 +2199,7 @@ public class CatalogOpExecutor {
public CreatePartitionRunnable(TableName tblName,
String partName, String cachePoolName, AtomicBoolean partitionCreated,
SettableFuture<Void> allFinished, AtomicInteger numPartitions,
List<Long> cacheDirIds) {
List<Long> cacheDirIds, Short cacheReplication) {
tblName_ = tblName;
partName_ = partName;
cachePoolName_ = cachePoolName;
@@ -2163,6 +2207,7 @@ public class CatalogOpExecutor {
allFinished_ = allFinished;
numPartitions_ = numPartitions;
cacheDirIds_ = cacheDirIds;
replication_ = cacheReplication;
}
public void run() {
@@ -2176,7 +2221,8 @@ public class CatalogOpExecutor {
if (cachePoolName_ != null) {
// Submit a new cache directive and update the partition metadata the
// directive id.
long id = HdfsCachingUtil.submitCachePartitionDirective(part, cachePoolName_);
long id = HdfsCachingUtil.submitCachePartitionDirective(
part, cachePoolName_, replication_);
synchronized (cacheDirIds_) {
cacheDirIds_.add(id);
}
@@ -2305,13 +2351,16 @@ public class CatalogOpExecutor {
// complete.
List<Long> cacheDirIds = Lists.<Long>newArrayList();
// If the table is cached, get its cache pool name. New partitions will inherit
// this property.
// If the table is cached, get its cache pool name and replication factor. New
// partitions will inherit this property.
String cachePoolName = null;
Long cacheDirId = HdfsCachingUtil.getCacheDirIdFromParams(
Short cacheReplication = 0;
Long cacheDirId = HdfsCachingUtil.getCacheDirectiveId(
table.getMetaStoreTable().getParameters());
if (cacheDirId != null) {
cachePoolName = HdfsCachingUtil.getCachePool(cacheDirId);
cacheReplication = HdfsCachingUtil.getCacheReplication(cacheDirId);
Preconditions.checkNotNull(cacheReplication);
if (table.getNumClusteringCols() == 0) cacheDirIds.add(cacheDirId);
}
@@ -2341,7 +2390,7 @@ public class CatalogOpExecutor {
// was written to the partition, a watch needs to be placed on the cache
// cache directive so the TableLoadingMgr can perform an async refresh once
// all data becomes cached.
cacheDirIds.add(HdfsCachingUtil.getCacheDirIdFromParams(
cacheDirIds.add(HdfsCachingUtil.getCacheDirectiveId(
partition.getMetaStorePartition().getParameters()));
}
if (partsToCreate.size() == 0) break;
@@ -2355,7 +2404,7 @@ public class CatalogOpExecutor {
Preconditions.checkState(partName != null && !partName.isEmpty());
CreatePartitionRunnable rbl = new CreatePartitionRunnable(tblName, partName,
cachePoolName, addedNewPartition, allFinished, numPartitions,
cacheDirIds);
cacheDirIds, cacheReplication);
executor_.execute(rbl);
}

View File

@@ -26,9 +26,13 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration;
import org.apache.log4j.Logger;
import com.cloudera.impala.analysis.TableName;
import com.cloudera.impala.catalog.HdfsPartition;
import com.cloudera.impala.common.FileSystemUtil;
import com.cloudera.impala.common.ImpalaException;
import com.cloudera.impala.common.ImpalaRuntimeException;
import com.cloudera.impala.thrift.JniCatalogConstants;
import com.cloudera.impala.thrift.THdfsCachingOp;
import com.google.common.base.Preconditions;
/**
@@ -40,6 +44,9 @@ public class HdfsCachingUtil {
// The key name used to save cache directive IDs in table/partition properties.
private final static String CACHE_DIR_ID_PROP_NAME = "cache_directive_id";
// The key name used to store the replication factor for cached files
private final static String CACHE_DIR_REPLICATION_PROP_NAME = "cache_replication";
// The number of caching refresh intervals that can go by when waiting for data to
// become cached before assuming no more progress is being made.
private final static int MAX_UNCHANGED_CACHING_REFRESH_INTERVALS = 5;
@@ -56,40 +63,39 @@ public class HdfsCachingUtil {
/**
* Caches the location of the given Hive Metastore Table and updates the
* table's properties with the submitted cache directive ID.
* table's properties with the submitted cache directive ID. The caller is
* responsible for not caching the same table twice, as HDFS will create a second
* cache directive even if it is similar to an already existing one.
*
* Returns the ID of the submitted cache directive and throws if there is an error
* submitting the directive or if the table was already cached.
* submitting.
*/
public static long submitCacheTblDirective(
org.apache.hadoop.hive.metastore.api.Table table,
String poolName) throws ImpalaRuntimeException {
if (table.getParameters().get(CACHE_DIR_ID_PROP_NAME) != null) {
throw new ImpalaRuntimeException(String.format(
"Table is already cached: %s.%s", table.getDbName(), table.getTableName()));
}
String poolName, short replication) throws ImpalaRuntimeException {
long id = HdfsCachingUtil.submitDirective(new Path(table.getSd().getLocation()),
poolName);
poolName, replication);
table.putToParameters(CACHE_DIR_ID_PROP_NAME, Long.toString(id));
table.putToParameters(CACHE_DIR_REPLICATION_PROP_NAME, Long.toString(replication));
return id;
}
/**
* Caches the location of the given Hive Metastore Partition and updates the
* partitions's properties with the submitted cache directive ID.
* partitions's properties with the submitted cache directive ID. The caller is
* responsible for not caching the same partition twice, as HDFS will create a second
* cache directive even if it is similar to an already existing one.
*
* Returns the ID of the submitted cache directive and throws if there is an error
* submitting the directive.
*/
public static long submitCachePartitionDirective(
org.apache.hadoop.hive.metastore.api.Partition part,
String poolName) throws ImpalaRuntimeException {
if (part.getParameters().get(CACHE_DIR_ID_PROP_NAME) != null) {
throw new ImpalaRuntimeException(String.format(
"Partition is already cached: %s.%s/%s", part.getDbName(), part.getTableName(),
part.getValues()));
}
String poolName, short replication) throws ImpalaRuntimeException {
long id = HdfsCachingUtil.submitDirective(new Path(part.getSd().getLocation()),
poolName);
poolName, replication);
part.putToParameters(CACHE_DIR_ID_PROP_NAME, Long.toString(id));
part.putToParameters(CACHE_DIR_REPLICATION_PROP_NAME, Long.toString(replication));
return id;
}
@@ -101,10 +107,11 @@ public class HdfsCachingUtil {
throws ImpalaRuntimeException {
Preconditions.checkNotNull(table);
LOG.debug("Uncaching table: " + table.getDbName() + "." + table.getTableName());
Long id = getCacheDirIdFromParams(table.getParameters());
Long id = getCacheDirectiveId(table.getParameters());
if (id == null) return;
HdfsCachingUtil.removeDirective(id);
table.getParameters().remove(CACHE_DIR_ID_PROP_NAME);
table.getParameters().remove(CACHE_DIR_REPLICATION_PROP_NAME);
}
/**
@@ -115,10 +122,11 @@ public class HdfsCachingUtil {
public static void uncachePartition(
org.apache.hadoop.hive.metastore.api.Partition part) throws ImpalaException {
Preconditions.checkNotNull(part);
Long id = getCacheDirIdFromParams(part.getParameters());
Long id = getCacheDirectiveId(part.getParameters());
if (id == null) return;
HdfsCachingUtil.removeDirective(id);
part.getParameters().remove(CACHE_DIR_ID_PROP_NAME);
part.getParameters().remove(CACHE_DIR_REPLICATION_PROP_NAME);
}
/**
@@ -126,7 +134,7 @@ public class HdfsCachingUtil {
* map. Returns null if the CACHE_DIR_ID_PROP_NAME key was not set or if
* there was an error parsing the associated ID.
*/
public static Long getCacheDirIdFromParams(Map<String, String> params) {
public static Long getCacheDirectiveId(Map<String, String> params) {
if (params == null) return null;
String idStr = params.get(CACHE_DIR_ID_PROP_NAME);
if (idStr == null) return null;
@@ -141,11 +149,39 @@ public class HdfsCachingUtil {
* Given a cache directive ID, returns the pool the directive is cached in.
* Returns null if no outstanding cache directive match this ID.
*/
public static String getCachePool(long requestId) throws ImpalaRuntimeException {
CacheDirectiveEntry entry = getDirective(requestId);
public static String getCachePool(long directiveId) throws ImpalaRuntimeException {
CacheDirectiveEntry entry = getDirective(directiveId);
return entry == null ? null : entry.getInfo().getPool();
}
/**
* Given a cache directive ID, returns the replication factor for the directive.
* Returns null if no outstanding cache directives match this ID.
*/
public static Short getCacheReplication(long directiveId)
throws ImpalaRuntimeException {
CacheDirectiveEntry entry = getDirective(directiveId);
return entry != null ? entry.getInfo().getReplication() : null;
}
/**
* Returns the cache replication value from the parameters map. We assume that only
* cached table parameters are used and the property is always present.
*/
public static Short getCachedCacheReplication(Map<String, String> params) {
Preconditions.checkNotNull(params);
String replication = params.get(CACHE_DIR_REPLICATION_PROP_NAME);
// For compatibility with tables created before allowing a custom replication factor
if (replication == null) {
return JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR;
}
try {
return Short.parseShort(replication);
} catch (NumberFormatException e) {
return JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR;
}
}
/**
* Waits on a cache directive to either complete or stop making progress. Progress is
* checked by polling the HDFS caching stats every
@@ -209,17 +245,18 @@ public class HdfsCachingUtil {
}
/**
* Submits a new caching directive for the specified cache pool name and path.
* Returns the directive ID if the submission was successful or an
* Submits a new caching directive for the specified cache pool name, path and
* replication. Returns the directive ID if the submission was successful or an
* ImpalaRuntimeException if the submission fails.
*/
private static long submitDirective(Path path, String poolName)
private static long submitDirective(Path path, String poolName, short replication)
throws ImpalaRuntimeException {
Preconditions.checkNotNull(path);
Preconditions.checkState(poolName != null && !poolName.isEmpty());
CacheDirectiveInfo info = new CacheDirectiveInfo.Builder()
.setExpiration(Expiration.NEVER)
.setPool(poolName)
.setReplication(replication)
.setPath(path).build();
LOG.debug("Submitting cache directive: " + info.toString());
try {
@@ -229,6 +266,59 @@ public class HdfsCachingUtil {
}
}
/**
* Update cache directive for a table and updates the metastore parameters.
* Returns the cache directive ID
*/
public static long modifyCacheDirective(Long id,
org.apache.hadoop.hive.metastore.api.Table table,
String poolName, short replication) throws ImpalaRuntimeException {
Preconditions.checkNotNull(id);
HdfsCachingUtil.modifyCacheDirective(id, new Path(table.getSd().getLocation()),
poolName, replication);
table.putToParameters(CACHE_DIR_ID_PROP_NAME, Long.toString(id));
table.putToParameters(CACHE_DIR_REPLICATION_PROP_NAME, Long.toString(replication));
return id;
}
/**
* Update cache directive for a partition and update the metastore parameters.
* Returns the cache directive ID
*/
public static long modifyCacheDirective(Long id,
org.apache.hadoop.hive.metastore.api.Partition part,
String poolName, short replication) throws ImpalaRuntimeException {
Preconditions.checkNotNull(id);
HdfsCachingUtil.modifyCacheDirective(id, new Path(part.getSd().getLocation()),
poolName, replication);
part.putToParameters(CACHE_DIR_ID_PROP_NAME, Long.toString(id));
part.putToParameters(CACHE_DIR_REPLICATION_PROP_NAME, Long.toString(replication));
return id;
}
/**
* Update an existing cache directive to avoid having the same entry multiple
* times
*/
private static void modifyCacheDirective(Long id, Path path, String poolName,
short replication) throws ImpalaRuntimeException {
Preconditions.checkNotNull(path);
Preconditions.checkNotNull(id);
Preconditions.checkState(poolName != null && !poolName.isEmpty());
CacheDirectiveInfo info = new CacheDirectiveInfo.Builder()
.setId(id)
.setExpiration(Expiration.NEVER)
.setPool(poolName)
.setReplication(replication)
.setPath(path).build();
LOG.debug("Modifying cache directive: " + info.toString());
try {
dfs.modifyCacheDirective(info);
} catch (IOException e) {
throw new ImpalaRuntimeException(e.getMessage(), e);
}
}
/**
* Removes the given cache directive if it exists, uncaching the data. If the
* cache request does not exist in HDFS no error is returned.
@@ -268,4 +358,77 @@ public class HdfsCachingUtil {
}
return null;
}
}
/**
* Check if the poolName matches the pool of the cache directive
* identified by directiveId
*/
public static boolean isSamePool(String poolName, Long directiveId)
throws ImpalaRuntimeException {
return poolName.equals(getCachePool(directiveId));
}
/**
* Helper method for frequent lookup of replication factor in the thrift caching
* structure.
*/
public static short getReplicationOrDefault(THdfsCachingOp op) {
return op.isSetReplication() ? op.getReplication() :
JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR;
}
/**
* Returns a boolean indicating if the given thrift caching operation would perform an
* update on an already existing cache directive.
*/
public static boolean isUpdateOp(THdfsCachingOp op, Map<String, String> params)
throws ImpalaRuntimeException {
Long directiveId = Long.parseLong(params.get(CACHE_DIR_ID_PROP_NAME));
CacheDirectiveEntry entry = getDirective(directiveId);
Preconditions.checkNotNull(entry);
// Verify cache pool
if (!op.getCache_pool_name().equals(entry.getInfo().getPool())) {
return false;
}
// Check cache replication factor
if ((op.isSetReplication() && op.getReplication() !=
entry.getInfo().getReplication()) || ( !op.isSetReplication() &&
entry.getInfo().getReplication() !=
JniCatalogConstants.HDFS_DEFAULT_CACHE_REPLICATION_FACTOR)) {
return true;
}
return false;
}
/**
* Validates the properties of the chosen cache pool. Throws on error.
*/
public static void validateCachePool(THdfsCachingOp op, Long directiveId,
TableName table, HdfsPartition partition) throws ImpalaRuntimeException {
CacheDirectiveEntry entry = getDirective(directiveId);
Preconditions.checkNotNull(entry);
if (!op.getCache_pool_name().equals(entry.getInfo().getPool())) {
throw new ImpalaRuntimeException(String.format("Cannot cache partition in " +
"pool '%s' because it is already cached in '%s'. To change the cache " +
"pool for this partition, first uncache using: ALTER TABLE %s.%s " +
"%sSET UNCACHED", op.getCache_pool_name(),
entry.getInfo().getPool(), table.getDb(), table,
// Insert partition string if partition non null
partition != null ? String.format(" PARTITION(%s) ",
partition.getPartitionName().replaceAll("/", ", ")) : ""));
}
}
/**
* Validates the properties of the chosen cache pool. Throws on error.
*/
public static void validateCachePool(THdfsCachingOp op, Long directiveId,
TableName table) throws ImpalaRuntimeException {
validateCachePool(op, directiveId, table, null);
}
}

View File

@@ -168,6 +168,7 @@ import com.cloudera.impala.analysis.SqlParserSymbols;
keywordMap.put("regexp", new Integer(SqlParserSymbols.KW_REGEXP));
keywordMap.put("rename", new Integer(SqlParserSymbols.KW_RENAME));
keywordMap.put("replace", new Integer(SqlParserSymbols.KW_REPLACE));
keywordMap.put("replication", new Integer(SqlParserSymbols.KW_REPLICATION));
keywordMap.put("returns", new Integer(SqlParserSymbols.KW_RETURNS));
keywordMap.put("revoke", new Integer(SqlParserSymbols.KW_REVOKE));
keywordMap.put("right", new Integer(SqlParserSymbols.KW_RIGHT));

View File

@@ -24,9 +24,9 @@ import junit.framework.Assert;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import com.cloudera.impala.catalog.CatalogException;
@@ -152,6 +152,8 @@ public class AnalyzeDDLTest extends AnalyzerTest {
// Caching ops
AnalyzesOk("alter table functional.alltypes add " +
"partition(year=2050, month=10) cached in 'testPool'");
AnalyzesOk("alter table functional.alltypes add " +
"partition(year=2050, month=10) cached in 'testPool' with replication = 10");
AnalyzesOk("alter table functional.alltypes add " +
"partition(year=2050, month=10) uncached");
AnalysisError("alter table functional.alltypes add " +
@@ -446,6 +448,18 @@ public class AnalyzeDDLTest extends AnalyzerTest {
AnalyzesOk("alter table functional.alltypes partition(year=2010, month=12) " +
"set cached in 'testPool'");
// Replication factor
AnalyzesOk("alter table functional.alltypes set cached in 'testPool' " +
"with replication = 10");
AnalyzesOk("alter table functional.alltypes partition(year=2010, month=12) " +
"set cached in 'testPool' with replication = 4");
AnalysisError("alter table functional.alltypes set cached in 'testPool' " +
"with replication = 0",
"Cache replication factor must be between 0 and Short.MAX_VALUE");
AnalysisError("alter table functional.alltypes set cached in 'testPool' " +
"with replication = 90000",
"Cache replication factor must be between 0 and Short.MAX_VALUE");
// Attempt to alter a table that is not backed by HDFS.
AnalysisError("alter table functional_hbase.alltypesnopart set cached in 'testPool'",
"ALTER TABLE SET not currently supported on HBase tables.");

View File

@@ -1774,9 +1774,14 @@ public class ParserTest {
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) CACHED 'pool'");
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) CACHED IN");
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) CACHED");
ParsesOk("ALTER TABLE Foo ADD PARTITION (j=2) CACHED IN 'pool' WITH replication = 3");
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) CACHED IN 'pool' " +
"with replication = -1");
ParsesOk("ALTER TABLE Foo ADD PARTITION (j=2) UNCACHED");
ParsesOk("ALTER TABLE Foo ADD PARTITION (j=2) LOCATION 'a/b' UNCACHED");
ParsesOk("ALTER TABLE Foo ADD PARTITION (j=2) LOCATION 'a/b' CACHED IN 'pool'");
ParsesOk("ALTER TABLE Foo ADD PARTITION (j=2) LOCATION 'a/b' CACHED IN 'pool' " +
"with replication = 3");
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) CACHED IN 'pool' LOCATION 'a/b'");
ParserError("ALTER TABLE Foo ADD PARTITION (j=2) UNCACHED LOCATION 'a/b'");
@@ -1907,7 +1912,8 @@ public class ParserTest {
}
}
for (String cacheClause: Lists.newArrayList("UNCACHED", "CACHED in 'pool'")) {
for (String cacheClause: Lists.newArrayList("UNCACHED", "CACHED in 'pool'",
"CACHED in 'pool' with replication = 4")) {
ParsesOk("ALTER TABLE Foo SET " + cacheClause);
ParsesOk("ALTER TABLE Foo PARTITION(j=0) SET " + cacheClause);
ParserError("ALTER TABLE Foo PARTITION(j=0) " + cacheClause);
@@ -2056,6 +2062,8 @@ public class ParserTest {
ParserError("CREATE TABLE Foo (d double) LOCATION 'a' COMMENT 'c'");
ParserError("CREATE TABLE Foo (d double) UNCACHED LOCATION '/a/b'");
ParserError("CREATE TABLE Foo (d double) CACHED IN 'pool' LOCATION '/a/b'");
ParserError("CREATE TABLE Foo (d double) CACHED IN 'pool' REPLICATION = 8 " +
"LOCATION '/a/b'");
ParserError("CREATE TABLE Foo (d double) LOCATION 'a' COMMENT 'c' STORED AS RCFILE");
ParserError("CREATE TABLE Foo (d double) LOCATION 'a' STORED AS RCFILE");
ParserError("CREATE TABLE Foo (d double) TBLPROPERTIES('a'='b') LOCATION 'a'");
@@ -2070,7 +2078,10 @@ public class ParserTest {
// Caching
ParsesOk("CREATE TABLE Foo (i int) CACHED IN 'myPool'");
ParsesOk("CREATE TABLE Foo (i int) CACHED IN 'myPool' WITH REPLICATION = 4");
ParsesOk("CREATE TABLE Foo (i int) PARTITIONED BY(j int) CACHED IN 'myPool'");
ParsesOk("CREATE TABLE Foo (i int) PARTITIONED BY(j int) CACHED IN 'myPool'" +
" WITH REPLICATION = 4");
ParsesOk("CREATE TABLE Foo (i int) PARTITIONED BY(j int) CACHED IN 'myPool'");
ParsesOk("CREATE TABLE Foo (i int) PARTITIONED BY(j int) LOCATION '/a' " +
"CACHED IN 'myPool'");
@@ -2080,6 +2091,10 @@ public class ParserTest {
ParserError("CREATE TABLE Foo (i int) IN 'myPool'");
ParserError("CREATE TABLE Foo (i int) PARTITIONED BY(j int) CACHED IN 'myPool' " +
"LOCATION '/a'");
ParserError("CREATE TABLE Foo (i int) CACHED IN 'myPool' WITH REPLICATION = -1");
ParserError("CREATE TABLE Foo (i int) CACHED IN 'myPool' WITH REPLICATION = 1.0");
ParserError("CREATE TABLE Foo (i int) CACHED IN 'myPool' " +
"WITH REPLICATION = cast(1 as double)");
// Invalid syntax
ParserError("CREATE TABLE IF EXISTS Foo.Bar (i int)");

View File

@@ -564,13 +564,13 @@ int,bigint
# Show the table stats before altering.
show table stats alltypes_test
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','4',-1,1,regex:.+KB,'NOT CACHED','SEQUENCE_FILE','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','RC_FILE','false'
'Total','',-1,2,regex:.+KB,'0B','',''
'2009','4',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','SEQUENCE_FILE','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','RC_FILE','false'
'Total','',-1,2,regex:.+KB,'0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Test altering the 'numRows' table property of a table.
@@ -587,13 +587,13 @@ set tblproperties ('numRows'='30', 'STATS_GENERATED_VIA_STATS_TASK'='true')
# Show the table stats after altering the table and partition stats.
show table stats alltypes_test
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','4',30,1,regex:.+KB,'NOT CACHED','SEQUENCE_FILE','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','RC_FILE','false'
'Total','',200,2,regex:.+KB,'0B','',''
'2009','4',30,1,regex:.+KB,'NOT CACHED','NOT CACHED','SEQUENCE_FILE','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','RC_FILE','false'
'Total','',200,2,regex:.+KB,'0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# IMPALA-1016: Testing scanning newly added columns

View File

@@ -13,12 +13,12 @@ compute stats compute_stats_db.decimal_tbl
---- QUERY
show table stats compute_stats_db.decimal_tbl
---- LABELS
d6, #Rows, #Files, Size, Bytes Cached, Format, Incremental Stats
d6, #Rows, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental Stats
---- RESULTS
'1',5,1,'375B','NOT CACHED','TEXT','false'
'Total',5,1,'375B','0B','',''
'1',5,1,'375B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',5,1,'375B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.decimal_tbl
@@ -47,11 +47,11 @@ compute stats compute_stats_db.mixed_types
---- QUERY
show table stats compute_stats_db.mixed_types
---- LABELS
#Rows, #Files, Size, Bytes Cached, Format, Incremental Stats
#Rows, #Files, Size, Bytes Cached, Cache Replication, Format, Incremental Stats
---- RESULTS
2,1,regex:.+B,'NOT CACHED','PARQUET','false'
2,1,regex:.+B,'NOT CACHED','NOT CACHED','PARQUET','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.mixed_types

View File

@@ -15,33 +15,33 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes_incremental
---- RESULTS
'2009','1',310,1,'24.56KB','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'Total','',7300,24,'586.84KB','0B','',''
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'Total','',7300,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING,STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes_incremental
@@ -70,33 +70,33 @@ drop incremental stats compute_stats_db.alltypes_incremental partition(year=2010
---- QUERY
show table stats compute_stats_db.alltypes_incremental;
---- RESULTS
'2009','1',310,1,'24.56KB','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','12',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'Total','',7300,24,'586.84KB','0B','',''
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','12',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',7300,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING,STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING,STRING, STRING
====
---- QUERY
compute incremental stats compute_stats_db.alltypes_incremental
@@ -108,33 +108,33 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes_incremental;
---- RESULTS
'2009','1',310,1,'24.56KB','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'Total','',7300,24,'586.84KB','0B','',''
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','1',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'Total','',7300,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes_incremental
@@ -171,10 +171,10 @@ STRING
---- QUERY
show table stats compute_stats_db.incremental_empty_partitioned;
---- RESULTS
'1',0,0,'0B','NOT CACHED','TEXT','true'
'Total',0,0,'0B','0B','',''
'1',0,0,'0B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',0,0,'0B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop incremental stats compute_stats_db.alltypes_incremental partition(year=2010, month=1);
@@ -190,33 +190,33 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes_incremental;
---- RESULTS
'2009','1',310,1,'24.56KB','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','1',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','2',280,1,'22.54KB','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','TEXT','true'
'Total','',6990,24,'586.84KB','0B','',''
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','1',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','true'
'2010','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','true'
'Total','',6990,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING,STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING,STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes_incremental
@@ -244,33 +244,33 @@ STRING, STRING, BIGINT, BIGINT, INT, DOUBLE
drop stats compute_stats_db.alltypes_incremental;
show table stats compute_stats_db.alltypes_incremental;
---- RESULTS
'2009','1',-1,1,'24.56KB','NOT CACHED','TEXT','false'
'2009','2',-1,1,'22.27KB','NOT CACHED','TEXT','false'
'2009','3',-1,1,'24.67KB','NOT CACHED','TEXT','false'
'2009','4',-1,1,'24.06KB','NOT CACHED','TEXT','false'
'2009','5',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','6',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','7',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','8',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','9',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','10',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','11',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','12',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','1',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','2',-1,1,'22.54KB','NOT CACHED','TEXT','false'
'2010','3',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','4',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','5',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','6',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','7',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','8',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','9',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','10',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','11',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','12',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'Total','',-1,24,'586.84KB','0B','',''
'2009','1',-1,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',-1,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','3',-1,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','4',-1,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','5',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','6',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','7',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','8',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','9',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','10',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','11',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','12',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','1',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',-1,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','3',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','4',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','5',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','6',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','7',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','8',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','9',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','10',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','11',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','12',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',-1,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Test that many partition keys work correctly
@@ -283,10 +283,10 @@ partition(p1=1, p2=2, p3=3, p4=4, p5=5, p6=6) values(1);
compute incremental stats compute_stats_db.incremental_many_part_keys;
show table stats compute_stats_db.incremental_many_part_keys;
---- RESULTS
'1','2','3','4','5','6',1,1,'2B','NOT CACHED','TEXT','true'
'Total','','','','','',1,1,'2B','0B','',''
'1','2','3','4','5','6',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total','','','','','',1,1,'2B','0B','','',''
---- TYPES
STRING, STRING, STRING, STRING, STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, STRING, STRING, STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
create table compute_stats_db.incremental_null_part_key(col int) partitioned by (p int);
@@ -297,11 +297,11 @@ compute incremental stats compute_stats_db.incremental_null_part_key partition(p
compute incremental stats compute_stats_db.incremental_null_part_key partition(p=NULL);
show table stats compute_stats_db.incremental_null_part_key;
---- RESULTS
'NULL',1,1,'2B','NOT CACHED','TEXT','true'
'2',1,1,'2B','NOT CACHED','TEXT','true'
'Total',2,2,'4B','0B','',''
'NULL',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'2',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',2,2,'4B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Check that incremental stats queries handle partitions with keyword names
@@ -312,20 +312,20 @@ compute incremental stats compute_stats_db.incremental_keyword_part_key
partition(`date`=1);
show table stats compute_stats_db.incremental_keyword_part_key;
---- RESULTS
'1',1,1,'2B','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','',''
'1',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop stats compute_stats_db.incremental_keyword_part_key;
compute incremental stats compute_stats_db.incremental_keyword_part_key;
show table stats compute_stats_db.incremental_keyword_part_key;
---- RESULTS
'1',1,1,'2B','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','',''
'1',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
create table compute_stats_db.incremental_string_part_value(col int) partitioned by
@@ -336,10 +336,10 @@ compute incremental stats compute_stats_db.incremental_string_part_value
partition(p="test_string");
show table stats compute_stats_db.incremental_string_part_value;
---- RESULTS
'test_string',1,1,'2B','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','',''
'test_string',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',1,1,'2B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
compute incremental stats functional_hbase.alltypes;
@@ -360,10 +360,10 @@ alter table compute_stats_db.incremental_drop_column drop column b;
compute incremental stats compute_stats_db.incremental_drop_column;
show table stats compute_stats_db.incremental_drop_column;
---- RESULTS
'1',1,1,'6B','NOT CACHED','TEXT','true'
'Total',1,1,'6B','0B','',''
'1',1,1,'6B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',1,1,'6B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Check that adding a column invalidates all incremental stats
@@ -375,20 +375,20 @@ alter table compute_stats_db.incremental_add_column add columns (c int);
compute incremental stats compute_stats_db.incremental_add_column;
show table stats compute_stats_db.incremental_add_column;
---- RESULTS
'1',2,2,'4B','NOT CACHED','TEXT','true'
'2',1,1,'2B','NOT CACHED','TEXT','true'
'Total',3,3,'6B','0B','',''
'1',2,2,'4B','NOT CACHED','NOT CACHED','TEXT','true'
'2',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','true'
'Total',3,3,'6B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
create table compute_stats_db.incremental_no_partitions (col int) partitioned by (p int);
compute incremental stats compute_stats_db.incremental_no_partitions;
show table stats compute_stats_db.incremental_no_partitions;
---- RESULTS
'Total',0,0,'0B','0B','',''
'Total',0,0,'0B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
create table compute_stats_db.incremental_not_partitioned (col int);
@@ -396,7 +396,7 @@ insert into compute_stats_db.incremental_not_partitioned values(1),(2);
compute incremental stats compute_stats_db.incremental_not_partitioned;
show table stats compute_stats_db.incremental_not_partitioned;
---- RESULTS
2,1,'4B','NOT CACHED','TEXT','false'
2,1,'4B','NOT CACHED','NOT CACHED','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====

View File

@@ -15,35 +15,35 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',310,1,'24.56KB','NOT CACHED','TEXT','false'
'2009','2',280,1,'22.27KB','NOT CACHED','TEXT','false'
'2009','3',310,1,'24.67KB','NOT CACHED','TEXT','false'
'2009','4',300,1,'24.06KB','NOT CACHED','TEXT','false'
'2009','5',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','6',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','7',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','8',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','9',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','10',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','11',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','12',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','1',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','2',280,1,'22.54KB','NOT CACHED','TEXT','false'
'2010','3',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','4',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','5',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','6',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','7',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','8',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','9',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','10',310,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','11',300,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','12',310,1,'24.97KB','NOT CACHED','TEXT','false'
'Total','',7300,24,'586.84KB','0B','',''
'2009','1',310,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',280,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','3',310,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','4',300,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','1',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',280,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','3',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','4',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','5',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','6',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','7',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','8',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','9',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','10',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','11',300,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','12',310,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',7300,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes
@@ -73,35 +73,35 @@ drop stats compute_stats_db.alltypes
---- QUERY
show table stats compute_stats_db.alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',-1,1,'24.56KB','NOT CACHED','TEXT','false'
'2009','10',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','11',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','12',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','2',-1,1,'22.27KB','NOT CACHED','TEXT','false'
'2009','3',-1,1,'24.67KB','NOT CACHED','TEXT','false'
'2009','4',-1,1,'24.06KB','NOT CACHED','TEXT','false'
'2009','5',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','6',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2009','7',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','8',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2009','9',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','1',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','10',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','11',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','12',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','2',-1,1,'22.54KB','NOT CACHED','TEXT','false'
'2010','3',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','4',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','5',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','6',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'2010','7',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','8',-1,1,'24.97KB','NOT CACHED','TEXT','false'
'2010','9',-1,1,'24.16KB','NOT CACHED','TEXT','false'
'Total','',-1,24,'586.84KB','0B','',''
'2009','1',-1,1,'24.56KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','10',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','11',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','12',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',-1,1,'22.27KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','3',-1,1,'24.67KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','4',-1,1,'24.06KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','5',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','6',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','7',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','8',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','9',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','1',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','10',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','11',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','12',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',-1,1,'22.54KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','3',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','4',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','5',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','6',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','7',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','8',-1,1,'24.97KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','9',-1,1,'24.16KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',-1,24,'586.84KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Note - the NDV for partition columns is read from the table metadata.
@@ -249,11 +249,11 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypesnopart
---- LABELS
#ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
100,3,'7.73KB','NOT CACHED','TEXT','false'
100,3,'7.73KB','NOT CACHED','NOT CACHED','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypesnopart
@@ -291,35 +291,35 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes_parquet
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','2',280,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','3',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','4',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','5',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','6',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','7',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','8',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','9',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','10',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','11',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','12',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','1',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','2',280,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','3',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','4',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','5',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','6',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','7',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','8',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','9',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','10',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','11',300,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','12',310,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'Total','',7300,24,regex:.+KB,'0B','',''
'2009','1',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','2',280,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','3',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','4',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','5',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','6',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','7',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','8',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','9',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','10',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','11',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','12',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','1',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','2',280,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','3',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','4',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','5',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','6',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','7',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','8',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','9',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','10',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','11',300,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','12',310,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'Total','',7300,24,regex:.+KB,'0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes_parquet
@@ -446,11 +446,11 @@ STRING
---- QUERY
show table stats compute_stats_db.alltypes_empty
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.alltypes_empty
@@ -542,11 +542,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_hive_alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_hive_alltypes
@@ -580,11 +580,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_hive_alltypes_extra_coldef
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_hive_alltypes_extra_coldef
@@ -619,11 +619,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_hive_alltypes_missing_coldef
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_hive_alltypes_missing_coldef
@@ -656,11 +656,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_hive_alltypes_type_mismatch
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_hive_alltypes_type_mismatch
@@ -701,11 +701,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_impala_alltypes_no_coldefs
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_impala_alltypes_no_coldefs
@@ -750,11 +750,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_impala_alltypes_bad_colname
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_impala_alltypes_bad_colname
@@ -799,11 +799,11 @@ STRING
---- QUERY
show table stats compute_stats_db.avro_impala_alltypes_bad_coltype
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'Total','',0,0,'0B','0B','',''
'Total','',0,0,'0B','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
show column stats compute_stats_db.avro_impala_alltypes_bad_coltype
@@ -863,10 +863,10 @@ STRING
---- QUERY
show table stats compute_stats_db.empty_partitioned
---- RESULTS
'1',0,0,'0B','NOT CACHED','TEXT','false'
'Total',0,0,'0B','0B','',''
'1',0,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',0,0,'0B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Insert non empty partition to the table with empty partition.
@@ -884,11 +884,11 @@ STRING
---- QUERY
show table stats compute_stats_db.empty_partitioned
---- RESULTS
'1',0,0,'0B','NOT CACHED','TEXT','false'
'2',1,1,'2B','NOT CACHED','TEXT','false'
'Total',1,1,'2B','0B','',''
'1',0,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',1,1,'2B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Verify partition stats work with empty and non-empty partition.
@@ -902,9 +902,9 @@ STRING
---- QUERY
show table stats compute_stats_db.empty_partitioned
---- RESULTS
'1',0,0,'0B','NOT CACHED','TEXT','false'
'2',1,1,'2B','NOT CACHED','TEXT','false'
'Total',1,1,'2B','0B','',''
'1',0,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',1,1,'2B','0B','','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====

View File

@@ -273,11 +273,11 @@ STRING, STRING, STRING
---- QUERY
show table stats like_view
---- LABELS
#ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
-1,0,'0B','NOT CACHED','TEXT','false'
-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table like_view
@@ -290,11 +290,11 @@ create table like_view_parquet like functional.view_view stored as parquet
---- QUERY
show table stats like_view_parquet
---- LABELS
#ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
-1,0,'0B','NOT CACHED','PARQUET','false'
-1,0,'0B','NOT CACHED','NOT CACHED','PARQUET','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table like_view_parquet

View File

@@ -21,21 +21,53 @@ select * from cached_tbl_nopart
INT
====
---- QUERY
# Unpartitioned table shows correct caching
show table stats cached_tbl_nopart
---- RESULTS
-1,1,'2B',regex:.+B,'TEXT'
-1,1,'2B',regex:.+B,'1','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Unpartitioned table is no longer cached
alter table cached_tbl_nopart set uncached
====
---- QUERY
show table stats cached_tbl_nopart
---- RESULTS
-1,1,'2B','NOT CACHED','TEXT'
-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table if exists cached_tbl_part
====
---- QUERY
create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool' with replication = 9
====
---- QUERY
# new partition should inherit the cached property
alter table cached_tbl_part add partition (j=0)
====
---- QUERY
# should be able to override the inherited cached property
alter table cached_tbl_part add partition (j=1) uncached
====
---- QUERY
alter table cached_tbl_part add partition (j=2) cached in 'testPool'
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'9','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B',regex:.+B,'9','TEXT','false'
'Total',-1,0,'0B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
drop table if exists cached_tbl_part
====
---- QUERY
create table cached_tbl_part (i int) partitioned by (j int) cached in 'testPool'
@@ -54,12 +86,12 @@ alter table cached_tbl_part add partition (j=2) cached in 'testPool'
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B','NOT CACHED','TEXT'
2,-1,0,'0B',regex:.+B,'TEXT'
Total,-1,0,'0B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B',regex:.+B,'1','TEXT','false'
'Total',-1,0,'0B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# uncache one of the partitions
@@ -68,12 +100,12 @@ alter table cached_tbl_part partition (j=2) set uncached
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B','NOT CACHED','TEXT'
2,-1,0,'0B','NOT CACHED','TEXT'
Total,-1,0,'0B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',-1,0,'0B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can uncache the same partition twice without an error.
@@ -82,12 +114,12 @@ alter table cached_tbl_part partition (j=2) set uncached
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B','NOT CACHED','TEXT'
2,-1,0,'0B','NOT CACHED','TEXT'
Total,-1,0,'0B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',-1,0,'0B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# mark an uncached partition as cached
@@ -96,12 +128,12 @@ alter table cached_tbl_part partition (j=1) set cached in 'testPool'
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B',regex:.+B,'TEXT'
2,-1,0,'0B','NOT CACHED','TEXT'
Total,-1,0,'0B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B',regex:.+B,'1','TEXT','false'
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',-1,0,'0B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# dynamic partition insert inherits table properties
@@ -111,16 +143,20 @@ j=3/: 1
j=4/: 1
====
---- QUERY
# Modify partition cache replication
alter table cached_tbl_part partition (j=3) set cached in 'testPool' with replication = 4
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B',regex:.+B,'TEXT'
2,-1,0,'0B','NOT CACHED','TEXT'
3,-1,1,'2B',regex:.+B,'TEXT'
4,-1,1,'2B',regex:.+B,'TEXT'
Total,-1,2,'4B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B',regex:.+B,'1','TEXT','false'
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'3',-1,1,'2B',regex:.+B,'4','TEXT','false'
'4',-1,1,'2B',regex:.+B,'1','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Set uncached clears all cache requests
@@ -130,14 +166,14 @@ alter table cached_tbl_part set uncached
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B','NOT CACHED','TEXT'
1,-1,0,'0B','NOT CACHED','TEXT'
2,-1,0,'0B','NOT CACHED','TEXT'
3,-1,1,'2B','NOT CACHED','TEXT'
4,-1,1,'2B','NOT CACHED','TEXT'
Total,-1,2,'4B',regex:.+B,''
'0',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'3',-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false'
'4',-1,1,'2B','NOT CACHED','NOT CACHED','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can call set uncached multiple times on the same partitioned table
@@ -152,14 +188,14 @@ alter table cached_tbl_part set cached in 'testPool'
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B',regex:.+B,'TEXT'
2,-1,0,'0B',regex:.+B,'TEXT'
3,-1,1,'2B',regex:.+B,'TEXT'
4,-1,1,'2B',regex:.+B,'TEXT'
Total,-1,2,'4B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B',regex:.+B,'1','TEXT','false'
'2',-1,0,'0B',regex:.+B,'1','TEXT','false'
'3',-1,1,'2B',regex:.+B,'1','TEXT','false'
'4',-1,1,'2B',regex:.+B,'1','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Can call set cached multiple times on the same partitioned table.
@@ -169,12 +205,104 @@ alter table cached_tbl_part set cached in 'testPool'
---- QUERY
show partitions cached_tbl_part
---- RESULTS
0,-1,0,'0B',regex:.+B,'TEXT'
1,-1,0,'0B',regex:.+B,'TEXT'
2,-1,0,'0B',regex:.+B,'TEXT'
3,-1,1,'2B',regex:.+B,'TEXT'
4,-1,1,'2B',regex:.+B,'TEXT'
Total,-1,2,'4B',regex:.+B,''
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B',regex:.+B,'1','TEXT','false'
'2',-1,0,'0B',regex:.+B,'1','TEXT','false'
'3',-1,1,'2B',regex:.+B,'1','TEXT','false'
'4',-1,1,'2B',regex:.+B,'1','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
INT, BIGINT, BIGINT, STRING, STRING, STRING
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Modify partition cache replication in preparation for table level alteration
alter table cached_tbl_part partition (j=3) set cached in 'testPool' with replication = 4
====
---- QUERY
# Uncache a partition in preparation for table level alteration
alter table cached_tbl_part partition (j=1) set uncached
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'1','TEXT','false'
'1',-1,0,'0B','NOT CACHED','NOT CACHED','TEXT','false'
'2',-1,0,'0B',regex:.+B,'1','TEXT','false'
'3',-1,1,'2B',regex:.+B,'4','TEXT','false'
'4',-1,1,'2B',regex:.+B,'1','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Set replication factor for all partitions regardless of their current state
alter table cached_tbl_part set cached in 'testPool' with replication = 8
---- RESULTS
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false'
'1',-1,0,'0B',regex:.+B,'8','TEXT','false'
'2',-1,0,'0B',regex:.+B,'8','TEXT','false'
'3',-1,1,'2B',regex:.+B,'8','TEXT','false'
'4',-1,1,'2B',regex:.+B,'8','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Add partition with different replication
alter table cached_tbl_part add partition(j=5) cached in 'testPool' with replication = 3
---- RESULTS
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false'
'1',-1,0,'0B',regex:.+B,'8','TEXT','false'
'2',-1,0,'0B',regex:.+B,'8','TEXT','false'
'3',-1,1,'2B',regex:.+B,'8','TEXT','false'
'4',-1,1,'2B',regex:.+B,'8','TEXT','false'
'5',-1,0,'0B',regex:.+B,'3','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Change replication for a partition
alter table cached_tbl_part partition(j=2) set cached in 'testPool' with replication = 3
---- RESULTS
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false'
'1',-1,0,'0B',regex:.+B,'8','TEXT','false'
'2',-1,0,'0B',regex:.+B,'3','TEXT','false'
'3',-1,1,'2B',regex:.+B,'8','TEXT','false'
'4',-1,1,'2B',regex:.+B,'8','TEXT','false'
'5',-1,0,'0B',regex:.+B,'3','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Add partition with inherited replication from table
alter table cached_tbl_part add partition(j=6) cached in 'testPool'
---- RESULTS
====
---- QUERY
show partitions cached_tbl_part
---- RESULTS
'0',-1,0,'0B',regex:.+B,'8','TEXT','false'
'1',-1,0,'0B',regex:.+B,'8','TEXT','false'
'2',-1,0,'0B',regex:.+B,'3','TEXT','false'
'3',-1,1,'2B',regex:.+B,'8','TEXT','false'
'4',-1,1,'2B',regex:.+B,'8','TEXT','false'
'5',-1,0,'0B',regex:.+B,'3','TEXT','false'
'6',-1,0,'0B',regex:.+B,'8','TEXT','false'
'Total',-1,2,'4B',regex:.+B,'','',''
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====

View File

@@ -8,126 +8,126 @@ drop stats alltypes;
compute stats alltypes;
show table stats alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',310,1,'19.95KB','NOT CACHED','TEXT','false'
'2009','2',280,1,'18.12KB','NOT CACHED','TEXT','false'
'2009','3',310,1,'20.06KB','NOT CACHED','TEXT','false'
'2009','4',300,1,'19.61KB','NOT CACHED','TEXT','false'
'2009','5',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','6',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','7',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','8',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','9',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','10',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','11',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','12',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','1',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','2',280,1,'18.39KB','NOT CACHED','TEXT','false'
'2010','3',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','4',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','5',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','6',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','7',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','8',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','9',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','10',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','11',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','12',310,1,'20.36KB','NOT CACHED','TEXT','false'
'Total','',7300,24,'478.45KB','0B','',''
'2009','1',310,1,'19.95KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',280,1,'18.12KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','3',310,1,'20.06KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','4',300,1,'19.61KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','5',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','6',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','7',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','8',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','9',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','10',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','11',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','12',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','1',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',280,1,'18.39KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','3',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','4',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','5',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','6',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','7',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','8',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','9',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','10',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','11',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','12',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',7300,24,'478.45KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# SHOW PARTITIONS returns the same results as SHOW TABLE STATS.
show partitions alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',310,1,'19.95KB','NOT CACHED','TEXT','false'
'2009','2',280,1,'18.12KB','NOT CACHED','TEXT','false'
'2009','3',310,1,'20.06KB','NOT CACHED','TEXT','false'
'2009','4',300,1,'19.61KB','NOT CACHED','TEXT','false'
'2009','5',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','6',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','7',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','8',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','9',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','10',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2009','11',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2009','12',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','1',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','2',280,1,'18.39KB','NOT CACHED','TEXT','false'
'2010','3',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','4',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','5',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','6',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','7',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','8',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','9',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','10',310,1,'20.36KB','NOT CACHED','TEXT','false'
'2010','11',300,1,'19.71KB','NOT CACHED','TEXT','false'
'2010','12',310,1,'20.36KB','NOT CACHED','TEXT','false'
'Total','',7300,24,'478.45KB','0B','',''
'2009','1',310,1,'19.95KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',280,1,'18.12KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','3',310,1,'20.06KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','4',300,1,'19.61KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','5',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','6',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','7',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','8',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','9',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','10',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','11',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','12',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','1',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','2',280,1,'18.39KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','3',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','4',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','5',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','6',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','7',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','8',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','9',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','10',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','11',300,1,'19.71KB','NOT CACHED','NOT CACHED','TEXT','false'
'2010','12',310,1,'20.36KB','NOT CACHED','NOT CACHED','TEXT','false'
'Total','',7300,24,'478.45KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Stats on an unpartitioned Hdfs table stored as text
show table stats alltypesaggmultifilesnopart
---- LABELS
#ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
#ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
11000,4,'805.23KB','NOT CACHED','TEXT','false'
11000,4,'805.23KB','NOT CACHED','NOT CACHED','TEXT','false'
---- TYPES
BIGINT, BIGINT, STRING, STRING, STRING, STRING
BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Stats on an Hdfs with mixed partition formats
show table stats alltypesmixedformat
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',-1,1,'19.59KB','NOT CACHED','TEXT','false'
'2009','2',-1,1,'21.35KB','NOT CACHED','SEQUENCE_FILE','false'
'2009','3',-1,1,'17.42KB','NOT CACHED','RC_FILE','false'
'Total','',-1,3,'58.36KB','0B','',''
'2009','1',-1,1,'19.59KB','NOT CACHED','NOT CACHED','TEXT','false'
'2009','2',-1,1,'21.35KB','NOT CACHED','NOT CACHED','SEQUENCE_FILE','false'
'2009','3',-1,1,'17.42KB','NOT CACHED','NOT CACHED','RC_FILE','false'
'Total','',-1,3,'58.36KB','0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Stats on a table that has no statistics
show table stats functional_parquet.alltypes
---- LABELS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, FORMAT, INCREMENTAL STATS
YEAR, MONTH, #ROWS, #FILES, SIZE, BYTES CACHED, CACHE REPLICATION, FORMAT, INCREMENTAL STATS
---- RESULTS
'2009','1',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','2',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','3',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','4',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','6',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','7',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','8',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','9',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','10',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','11',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2009','12',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','1',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','2',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','3',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','4',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','5',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','6',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','7',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','8',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','9',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','10',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','11',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'2010','12',-1,1,regex:.+KB,'NOT CACHED','PARQUET','false'
'Total','',-1,24,regex:.+KB,'0B','',''
'2009','1',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','2',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','3',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','4',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','5',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','6',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','7',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','8',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','9',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','10',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','11',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2009','12',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','1',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','2',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','3',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','4',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','5',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','6',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','7',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','8',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','9',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','10',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','11',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'2010','12',-1,1,regex:.+KB,'NOT CACHED','NOT CACHED','PARQUET','false'
'Total','',-1,24,regex:.+KB,'0B','','',''
---- TYPES
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING
STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING
====
---- QUERY
# Stats on an HBase table

View File

@@ -148,7 +148,7 @@ class TestFetch(HS2TestSuite):
assert num_rows == 25
# Match whether stats are computed or not
assert re.match(
r"2009, 1, -?\d+, -?\d+, \d*\.?\d+KB, NOT CACHED, TEXT", result) is not None
r"2009, 1, -?\d+, -?\d+, \d*\.?\d+KB, NOT CACHED, NOT CACHED, TEXT", result) is not None
@needs_session()
def test_show_column_stats(self):

View File

@@ -98,7 +98,6 @@ class TestHdfsCachingDdl(ImpalaTestSuite):
v.get_value('table_format').compression_codec == 'none')
@pytest.mark.execute_serially
@pytest.mark.xfail(run=False, reason="IMPALA-1037. This test is flaky")
def test_caching_ddl(self, vector):
self.client.execute("drop table if exists functional.cached_tbl_part")
self.client.execute("drop table if exists functional.cached_tbl_nopart")
@@ -107,9 +106,9 @@ class TestHdfsCachingDdl(ImpalaTestSuite):
num_entries_pre = get_num_cache_requests()
self.run_test_case('QueryTest/hdfs-caching', vector)
# After running this test case we should be left with 6 cache requests.
# In this case, 1 for each table + 4 more for each cached partition.
assert num_entries_pre == get_num_cache_requests() - 6
# After running this test case we should be left with 8 cache requests.
# In this case, 1 for each table + 7 more for each cached partition.
assert num_entries_pre == get_num_cache_requests() - 8
self.client.execute("drop table functional.cached_tbl_part")
self.client.execute("drop table functional.cached_tbl_nopart")