diff --git a/be/src/service/impala-beeswax-server.cc b/be/src/service/impala-beeswax-server.cc index f28995da5..cdbffcd83 100644 --- a/be/src/service/impala-beeswax-server.cc +++ b/be/src/service/impala-beeswax-server.cc @@ -504,7 +504,6 @@ void ImpalaServer::ResetTable(impala::TStatus& status, const TResetTableReq& req Status ImpalaServer::QueryToTQueryContext(const Query& query, TQueryCtx* query_ctx) { - query_ctx->request.query_options = default_query_options_; query_ctx->request.stmt = query.query; VLOG_QUERY << "query: " << ThriftDebugString(query); { @@ -519,6 +518,7 @@ Status ImpalaServer::QueryToTQueryContext(const Query& query, lock_guard l(session->lock); if (session->connected_user.empty()) session->connected_user = query.hadoop_user; query_ctx->session.connected_user = session->connected_user; + query_ctx->request.query_options = session->default_query_options; } // Override default query options with Query.Configuration diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index d399f566b..715692680 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -1678,6 +1678,7 @@ void ImpalaServer::ConnectionStart( session_state->database = "default"; session_state->session_type = TSessionType::BEESWAX; session_state->network_address = connection_context.network_address; + session_state->default_query_options = default_query_options_; // If the username was set by a lower-level transport, use it. if (!connection_context.username.empty()) { session_state->connected_user = connection_context.username; diff --git a/be/src/service/query-exec-state.cc b/be/src/service/query-exec-state.cc index 1d5e59f17..0e3733851 100644 --- a/be/src/service/query-exec-state.cc +++ b/be/src/service/query-exec-state.cc @@ -158,6 +158,31 @@ Status ImpalaServer::QueryExecState::Exec(TExecRequest* exec_request) { exec_request_.query_options.sync_ddl)); return Status::OK; } + case TStmtType::SET: { + DCHECK(exec_request_.__isset.set_query_option_request); + lock_guard l(session_->lock); + if (exec_request_.set_query_option_request.__isset.key) { + // "SET key=value" updates the session query options. + DCHECK(exec_request_.set_query_option_request.__isset.value); + RETURN_IF_ERROR(parent_server_->SetQueryOptions( + exec_request_.set_query_option_request.key, + exec_request_.set_query_option_request.value, + &session_->default_query_options)); + } else { + // "SET" returns a table of all query options. + map config; + parent_server_->TQueryOptionsToMap( + session_->default_query_options, &config); + vector keys, values; + map::const_iterator itr = config.begin(); + for (; itr != config.end(); ++itr) { + keys.push_back(itr->first); + values.push_back(itr->second); + } + SetResultSet(keys, values); + } + return Status::OK; + } default: stringstream errmsg; errmsg << "Unknown exec request stmt type: " << exec_request_.stmt_type; diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift index 01f6b3a69..cef1b2ce8 100644 --- a/common/thrift/Frontend.thrift +++ b/common/thrift/Frontend.thrift @@ -365,6 +365,13 @@ struct TCatalogOpRequest { 10: optional CatalogObjects.TTableName show_create_table_params } +// Parameters for the SET query option command +struct TSetQueryOptionRequest { + // Set for "SET key=value", unset for "SET" statement. + 1: optional string key + 2: optional string value +} + // HiveServer2 Metadata operations (JniFrontend.hiveServer2MetadataOperation) enum TMetadataOpcode { GET_TYPE_INFO, @@ -442,6 +449,9 @@ struct TExecRequest { // List of warnings that were generated during analysis. May be empty. 9: required list analysis_warnings + + // Set iff stmt_type is SET + 10: optional TSetQueryOptionRequest set_query_option_request } // Parameters to FeSupport.cacheJar(). diff --git a/common/thrift/Types.thrift b/common/thrift/Types.thrift index 92791af55..2c1db2f2f 100644 --- a/common/thrift/Types.thrift +++ b/common/thrift/Types.thrift @@ -92,7 +92,8 @@ enum TStmtType { DDL, // Data definition, e.g. CREATE TABLE (includes read-only functions e.g. SHOW) DML, // Data modification e.g. INSERT EXPLAIN, - LOAD // Statement type for LOAD commands + LOAD, // Statement type for LOAD commands + SET } // Level of verboseness for "explain" output. diff --git a/fe/src/main/cup/sql-parser.y b/fe/src/main/cup/sql-parser.y index 9d8599cdf..f67b0eef7 100644 --- a/fe/src/main/cup/sql-parser.y +++ b/fe/src/main/cup/sql-parser.y @@ -49,19 +49,20 @@ parser code {: private final List expectedTokenIds_ = new ArrayList(); // to avoid reporting trivial tokens as expected tokens in error messages - private boolean reportExpectedToken(Integer tokenId) { + private boolean reportExpectedToken(Integer tokenId, int numExpectedTokens) { if (SqlScanner.isKeyword(tokenId) || tokenId.intValue() == SqlParserSymbols.COMMA || tokenId.intValue() == SqlParserSymbols.IDENT) { return true; } else { - return false; + // if this is the only valid token, always report it + return numExpectedTokens == 1; } } private String getErrorTypeMessage(int lastTokenId) { String msg = null; - switch(lastTokenId) { + switch (lastTokenId) { case SqlParserSymbols.UNMATCHED_STRING_LITERAL: msg = "Unmatched string literal"; break; @@ -208,7 +209,7 @@ parser code {: Integer tokenId = null; for (int i = 0; i < expectedTokenIds_.size(); ++i) { tokenId = expectedTokenIds_.get(i); - if (reportExpectedToken(tokenId)) { + if (reportExpectedToken(tokenId, expectedTokenIds_.size())) { expectedToken = SqlScanner.tokenIdMap.get(tokenId); result.append(expectedToken + ", "); } @@ -275,6 +276,7 @@ nonterminal List union_operand_list; nonterminal List values_operand_list; // USE stmt nonterminal UseStmt use_stmt; +nonterminal SetStmt set_stmt; nonterminal ShowTablesStmt show_tables_stmt; nonterminal ShowDbsStmt show_dbs_stmt; nonterminal ShowPartitionsStmt show_partitions_stmt; @@ -500,6 +502,8 @@ stmt ::= {: RESULT = load; :} | reset_metadata_stmt: reset_metadata {: RESULT = reset_metadata; :} + | set_stmt:set + {: RESULT = set; :} ; load_stmt ::= @@ -1457,6 +1461,15 @@ select_clause ::= :} ; +set_stmt ::= + KW_SET IDENT:key EQUAL literal:l + {: RESULT = new SetStmt(key, l.getStringValue()); :} + | KW_SET IDENT:key EQUAL IDENT:ident + {: RESULT = new SetStmt(key, ident); :} + | KW_SET + {: RESULT = new SetStmt(null, null); :} + ; + opt_straight_join ::= KW_STRAIGHT_JOIN {: RESULT = true; :} diff --git a/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java b/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java index 6d4471429..33b987711 100644 --- a/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java +++ b/fe/src/main/java/com/cloudera/impala/analysis/AnalysisContext.java @@ -73,6 +73,7 @@ public class AnalysisContext { public boolean isCreateDataSrcStmt() { return stmt_ instanceof CreateDataSrcStmt; } public boolean isLoadDataStmt() { return stmt_ instanceof LoadDataStmt; } public boolean isUseStmt() { return stmt_ instanceof UseStmt; } + public boolean isSetStmt() { return stmt_ instanceof SetStmt; } public boolean isShowTablesStmt() { return stmt_ instanceof ShowTablesStmt; } public boolean isShowDbsStmt() { return stmt_ instanceof ShowDbsStmt; } public boolean isShowDataSrcsStmt() { return stmt_ instanceof ShowDataSrcsStmt; } @@ -190,6 +191,11 @@ public class AnalysisContext { return (UseStmt) stmt_; } + public SetStmt getSetStmt() { + Preconditions.checkState(isSetStmt()); + return (SetStmt) stmt_; + } + public ShowTablesStmt getShowTablesStmt() { Preconditions.checkState(isShowTablesStmt()); return (ShowTablesStmt) stmt_; diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java new file mode 100644 index 000000000..9ad131b32 --- /dev/null +++ b/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java @@ -0,0 +1,52 @@ +// Copyright 2014 Cloudera Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.cloudera.impala.analysis; + +import com.cloudera.impala.thrift.TSetQueryOptionRequest; +import com.google.common.base.Preconditions; + +/** + * Representation of a SET query options statement. + */ +public class SetStmt extends StatementBase { + private final String key_; + private final String value_; + + public SetStmt(String key, String value) { + Preconditions.checkArgument((key == null) == (value == null)); + Preconditions.checkArgument(key == null || !key.isEmpty()); + key_ = key; + value_ = value; + } + + @Override + public String toSql() { + return "SET " + ToSqlUtils.getIdentSql(key_) + "='" + value_ + "'"; + } + + @Override + public void analyze(Analyzer analyzer) { + // Query option key is validated by the backend. + } + + public TSetQueryOptionRequest toThrift() { + TSetQueryOptionRequest request = new TSetQueryOptionRequest(); + if (key_ != null) { + request.setKey(key_); + request.setValue(value_); + } + return request; + } +} diff --git a/fe/src/main/java/com/cloudera/impala/service/Frontend.java b/fe/src/main/java/com/cloudera/impala/service/Frontend.java index 9555c5234..5f108b72b 100644 --- a/fe/src/main/java/com/cloudera/impala/service/Frontend.java +++ b/fe/src/main/java/com/cloudera/impala/service/Frontend.java @@ -647,6 +647,13 @@ public class Frontend { new TColumn("summary", Type.STRING.toThrift())))); result.setLoad_data_request(analysisResult.getLoadDataStmt().toThrift()); return result; + } else if (analysisResult.isSetStmt()) { + result.stmt_type = TStmtType.SET; + result.setResult_set_metadata(new TResultSetMetadata(Arrays.asList( + new TColumn("option", Type.STRING.toThrift()), + new TColumn("value", Type.STRING.toThrift())))); + result.setSet_query_option_request(analysisResult.getSetStmt().toThrift()); + return result; } // create TQueryExecRequest diff --git a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java index 9d182a5f2..00404b83f 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/AnalyzeStmtsTest.java @@ -2316,4 +2316,9 @@ public class AnalyzeStmtsTest extends AnalyzerTest { cl.getSimpleName(), expectedNumMembers, actualNumMembers)); } } + + @Test + public void TestSetQueryOption() { + AnalyzesOk("set foo=true"); + } } diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java index 73d245c21..50dc0225f 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ParserTest.java @@ -2147,7 +2147,7 @@ public class ParserTest { "^\n" + "Encountered: IDENTIFIER\n" + "Expected: ALTER, COMPUTE, CREATE, DESCRIBE, DROP, EXPLAIN, INSERT, " + - "INVALIDATE, LOAD, REFRESH, SELECT, SHOW, USE, VALUES, WITH\n"); + "INVALIDATE, LOAD, REFRESH, SELECT, SET, SHOW, USE, VALUES, WITH\n"); // missing select list ParserError("select from t", @@ -2283,6 +2283,14 @@ public class ParserTest { " ^\n" + "Encountered: EMPTY IDENTIFIER\n" + "Expected: IDENTIFIER\n"); + + // Expecting = token + ParserError("SET foo", + "Syntax error in line 1:\n" + + "SET foo\n" + + " ^\n" + + "Encountered: EOF\n" + + "Expected: =\n"); } @Test @@ -2399,4 +2407,26 @@ public class ParserTest { ParserError("SELECT a, count(*) FROM foo ORDER BY (SELECT) a FROM bar DESC"); ParserError("SELECT a, count(*) FROM foo ORDER BY (SELECT a FROM bar ASC"); } + + @Test + public void TestSet() { + ParsesOk("SET foo='bar'"); + ParsesOk("SET foo=\"bar\""); + ParsesOk("SET foo=bar"); + ParsesOk("SET foo = bar"); + ParsesOk("SET foo=1"); + ParsesOk("SET foo=true"); + ParsesOk("SET foo=false"); + ParsesOk("SET foo=1.2"); + ParsesOk("SET foo=null"); + ParsesOk("SET foo=10g"); + ParsesOk("SET `foo`=0"); + ParsesOk("SET foo=''"); + ParsesOk("SET"); + + ParserError("SET foo"); + ParserError("SET foo="); + ParserError("SET foo=1+2"); + ParserError("SET foo = '10"); + } } diff --git a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java index 803c66174..ee1492f7d 100644 --- a/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java +++ b/fe/src/test/java/com/cloudera/impala/analysis/ToSqlTest.java @@ -742,4 +742,13 @@ public class ToSqlTest extends AnalyzerTest { public void testDecimal() { testToSql("select cast(1 as decimal)", "SELECT CAST(1 AS DECIMAL(9,0))"); } + + /** + * Tests set query option statements are output correctly. + */ + @Test + public void testSet() { + testToSql("set a = 1", "SET a='1'"); + testToSql("set `a b` = \"x y\"", "SET `a b`='x y'"); + } } diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test new file mode 100644 index 000000000..12e02e9ca --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/set.test @@ -0,0 +1,100 @@ +==== +---- QUERY +set +---- RESULTS: VERIFY_IS_SUBSET +'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0' +'ABORT_ON_ERROR','0' +'ALLOW_UNSUPPORTED_FORMATS','0' +'BATCH_SIZE','0' +'DEBUG_ACTION','' +'DEFAULT_ORDER_BY_LIMIT','-1' +'DISABLE_CACHED_READS','0' +'DISABLE_CODEGEN','0' +'DISABLE_OUTERMOST_TOPN','0' +'EXPLAIN_LEVEL','1' +'HBASE_CACHE_BLOCKS','0' +'HBASE_CACHING','0' +'MAX_ERRORS','0' +'MAX_IO_BUFFERS','0' +'MAX_SCAN_RANGE_LENGTH','0' +'MEM_LIMIT','0' +'NUM_NODES','0' +'NUM_SCANNER_THREADS','0' +'PARQUET_COMPRESSION_CODEC','SNAPPY' +'PARQUET_FILE_SIZE','0' +'REQUEST_POOL','' +'RESERVATION_REQUEST_TIMEOUT','0' +'RM_INITIAL_MEM','0' +'SYNC_DDL','0' +'V_CPU_CORES','0' +---- TYPES +STRING, STRING +==== +---- QUERY +set explain_level=3 +==== +---- QUERY +set +---- RESULTS: VERIFY_IS_SUBSET +'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0' +'ABORT_ON_ERROR','0' +'ALLOW_UNSUPPORTED_FORMATS','0' +'BATCH_SIZE','0' +'DEBUG_ACTION','' +'DEFAULT_ORDER_BY_LIMIT','-1' +'DISABLE_CACHED_READS','0' +'DISABLE_CODEGEN','0' +'DISABLE_OUTERMOST_TOPN','0' +'EXPLAIN_LEVEL','3' +'HBASE_CACHE_BLOCKS','0' +'HBASE_CACHING','0' +'MAX_ERRORS','0' +'MAX_IO_BUFFERS','0' +'MAX_SCAN_RANGE_LENGTH','0' +'MEM_LIMIT','0' +'NUM_NODES','0' +'NUM_SCANNER_THREADS','0' +'PARQUET_COMPRESSION_CODEC','SNAPPY' +'PARQUET_FILE_SIZE','0' +'REQUEST_POOL','' +'RESERVATION_REQUEST_TIMEOUT','0' +'RM_INITIAL_MEM','0' +'SYNC_DDL','0' +'V_CPU_CORES','0' +---- TYPES +STRING, STRING +==== +---- QUERY +set explain_level='0' +==== +---- QUERY +set +---- RESULTS: VERIFY_IS_SUBSET +'ABORT_ON_DEFAULT_LIMIT_EXCEEDED','0' +'ABORT_ON_ERROR','0' +'ALLOW_UNSUPPORTED_FORMATS','0' +'BATCH_SIZE','0' +'DEBUG_ACTION','' +'DEFAULT_ORDER_BY_LIMIT','-1' +'DISABLE_CACHED_READS','0' +'DISABLE_CODEGEN','0' +'DISABLE_OUTERMOST_TOPN','0' +'EXPLAIN_LEVEL','0' +'HBASE_CACHE_BLOCKS','0' +'HBASE_CACHING','0' +'MAX_ERRORS','0' +'MAX_IO_BUFFERS','0' +'MAX_SCAN_RANGE_LENGTH','0' +'MEM_LIMIT','0' +'NUM_NODES','0' +'NUM_SCANNER_THREADS','0' +'PARQUET_COMPRESSION_CODEC','SNAPPY' +'PARQUET_FILE_SIZE','0' +'REQUEST_POOL','' +'RESERVATION_REQUEST_TIMEOUT','0' +'RM_INITIAL_MEM','0' +'SYNC_DDL','0' +'V_CPU_CORES','0' +---- TYPES +STRING, STRING +==== \ No newline at end of file diff --git a/tests/query_test/test_set.py b/tests/query_test/test_set.py new file mode 100644 index 000000000..605e0131d --- /dev/null +++ b/tests/query_test/test_set.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# Copyright (c) 2014 Cloudera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tests for SET + +import logging +import pytest +from tests.beeswax.impala_beeswax import ImpalaBeeswaxException +from tests.common.test_dimensions import * +from tests.common.impala_test_suite import ImpalaTestSuite, SINGLE_NODE_ONLY + +class TestSet(ImpalaTestSuite): + @classmethod + def get_workload(self): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestSet, cls).add_test_dimensions() + # This test only needs to be run once. + cls.TestMatrix.add_dimension(create_single_exec_option_dimension()) + cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload())) + + def test_set(self, vector): + self.run_test_case('QueryTest/set', vector) + + def test_set_negative(self, vector): + # Test that SET with an invalid config option name fails + try: + self.execute_query('set foo=bar') + assert False, 'Expected to fail' + except ImpalaBeeswaxException as e: + assert "Ignoring invalid configuration option: foo" in str(e) + + # Test that SET with an invalid config option value fails + try: + self.execute_query('set parquet_compression_codec=bar') + assert False, 'Expected to fail' + except ImpalaBeeswaxException as e: + assert "Invalid parquet compression codec: bar" in str(e) + + @pytest.mark.execute_serially + def test_set_mem_limit(self, vector): + # Test that SET actually does change the mem_limit + self.execute_query('select 1') + self.execute_query('set mem_limit=1'); + try: + self.execute_query('select 1') + assert False, 'Expected to fail' + except ImpalaBeeswaxException as e: + assert "Memory limit exceeded" in str(e) + self.execute_query('set mem_limit=0'); + self.execute_query('select 1')