diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert.test b/testdata/workloads/functional-query/queries/QueryTest/insert.test index e16bbf797..a91255308 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert.test @@ -10,8 +10,7 @@ where year=2009 and month=04 RESET alltypesnopart_insert RELOAD alltypesnopart_insert ---- RESULTS ----- PARTITIONS -/: 25 +: 25 ==== ---- QUERY # search the overwritten table to verify the results @@ -57,8 +56,7 @@ where year=2009 and month=04 ---- SETUP RESET alltypesnopart_insert ---- RESULTS ----- PARTITIONS -/: 25 +: 25 ==== ---- QUERY # search the table to verify it contains 25 rows @@ -80,8 +78,7 @@ where year=2009 and month=4 DROP PARTITIONS alltypesinsert RESET alltypesinsert ---- RESULTS ----- PARTITIONS -/year=2009/month=4/: 25 +year=2009/month=4/: 25 ==== ---- QUERY # search the overwritten partition to verify the results @@ -129,8 +126,7 @@ where year=2009 and month=4 ---- SETUP DROP PARTITIONS alltypesinsert ---- RESULTS ----- PARTITIONS -/year=2009/month=4/: 25 +year=2009/month=4/: 25 ==== ---- QUERY # search the partition to verify it contains all 25 rows @@ -152,10 +148,9 @@ where year=2009 and month>1 and month<=4 ---- SETUP DROP PARTITIONS alltypesinsert ---- RESULTS ----- PARTITIONS -/year=2009/month=2/: 25 -/year=2009/month=3/: 25 -/year=2009/month=4/: 25 +year=2009/month=2/: 25 +year=2009/month=3/: 25 +year=2009/month=4/: 25 ==== ---- QUERY # search the overwritten partition to verify the results @@ -253,13 +248,9 @@ where year=2009 and month>=1 and month<4 ---- SETUP DROP PARTITIONS alltypesinsert ---- RESULTS -year=2009/month=1/ -year=2009/month=2/ -year=2009/month=3/ ----- PARTITIONS -/year=2009/month=1/: 25 -/year=2009/month=2/: 25 -/year=2009/month=3/: 25 +year=2009/month=1/: 25 +year=2009/month=2/: 25 +year=2009/month=3/: 25 ==== ---- QUERY # search the partitions to verify they contain all 75 rows @@ -280,11 +271,10 @@ from functional.alltypessmall ---- SETUP DROP PARTITIONS alltypesinsert ---- RESULTS ----- PARTITIONS -/year=2009/month=1/: 25 -/year=2009/month=2/: 25 -/year=2009/month=3/: 25 -/year=2009/month=4/: 25 +year=2009/month=1/: 25 +year=2009/month=2/: 25 +year=2009/month=3/: 25 +year=2009/month=4/: 25 ==== ---- QUERY # search the overwritten partition to verify the results @@ -406,11 +396,10 @@ from functional.alltypessmall ---- SETUP DROP PARTITIONS alltypesinsert ---- RESULTS ----- PARTITIONS -/year=2009/month=1/: 25 -/year=2009/month=2/: 25 -/year=2009/month=3/: 25 -/year=2009/month=4/: 25 +year=2009/month=1/: 25 +year=2009/month=2/: 25 +year=2009/month=3/: 25 +year=2009/month=4/: 25 ==== ---- QUERY # search the partitions to verify they contain all 100 rows @@ -426,7 +415,7 @@ bigint INSERT INTO TABLE insert_string_partitioned PARTITION(s2="/\%.") SELECT "value" FROM alltypessmall LIMIT 1; ---- SETUP DROP PARTITIONS insert_string_partitioned ----- PARTITIONS +---- RESULTS s2=%2F%5C%25./: 1 ==== ---- QUERY diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert_null.test b/testdata/workloads/functional-query/queries/QueryTest/insert_null.test index bb262630b..1de2c7ca9 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert_null.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert_null.test @@ -6,8 +6,7 @@ select NULL, "", "NULL", "\\N", NULL from alltypes limit 1 ---- SETUP RESET nullinsert ---- RESULTS ----- PARTITIONS -/: 1 +: 1 ==== ---- QUERY select * from nullinsert diff --git a/testdata/workloads/functional-query/queries/QueryTest/insert_overwrite.test b/testdata/workloads/functional-query/queries/QueryTest/insert_overwrite.test index d02c37ffc..19d9cfdde 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/insert_overwrite.test +++ b/testdata/workloads/functional-query/queries/QueryTest/insert_overwrite.test @@ -7,8 +7,7 @@ from functional.tinyinttable ---- SETUP RESET insert_overwrite_nopart ---- RESULTS ----- PARTITIONS -/: 10 +: 10 ==== ---- QUERY # Check results - note larger limit than expected in case there's more data written than there should be @@ -36,8 +35,7 @@ insert overwrite table insert_overwrite_nopart select 10 from functional.tinyinttable ---- RESULTS ----- PARTITIONS -/: 10 +: 10 ==== ---- QUERY # check results from previous insert @@ -66,7 +64,6 @@ select 3 from functional.tinyinttable limit 0 ---- RESULTS ----- PARTITIONS ==== ---- QUERY select count(*) from insert_overwrite_nopart @@ -84,8 +81,7 @@ from functional.tinyinttable ---- SETUP RESET insert_overwrite_partitioned ---- RESULTS ----- PARTITIONS -/col2=5/: 10 +col2=5/: 10 ==== ---- QUERY # Check results of previous insert @@ -115,8 +111,7 @@ from functional.tinyinttable ---- SETUP RESET insert_overwrite_nopart ---- RESULTS ----- PARTITIONS -/col2=6/: 10 +col2=6/: 10 ==== ---- QUERY # Check results of previous insert @@ -154,8 +149,7 @@ insert overwrite table insert_overwrite_partitioned partition(col2=5) select 10 from functional.tinyinttable ---- RESULTS ----- PARTITIONS -/col2=5/: 10 +col2=5/: 10 ==== ---- QUERY # Confirm that one partition is still intact @@ -194,17 +188,16 @@ from functional.tinyinttable ---- SETUP DROP PARTITIONS insert_overwrite_partitioned ---- RESULTS ----- PARTITIONS -/col2=0/: 1 -/col2=1/: 1 -/col2=2/: 1 -/col2=3/: 1 -/col2=4/: 1 -/col2=5/: 1 -/col2=6/: 1 -/col2=7/: 1 -/col2=8/: 1 -/col2=9/: 1 +col2=0/: 1 +col2=1/: 1 +col2=2/: 1 +col2=3/: 1 +col2=4/: 1 +col2=5/: 1 +col2=6/: 1 +col2=7/: 1 +col2=8/: 1 +col2=9/: 1 ==== ---- QUERY # Confirm results of previous insert @@ -233,8 +226,7 @@ partition(col2) select 10, 0 from functional.tinyinttable where int_col = 0 ---- RESULTS ----- PARTITIONS -/col2=0/: 1 +col2=0/: 1 ==== ---- QUERY select col1, col2 diff --git a/testdata/workloads/tpch/queries/tpch-q11.test b/testdata/workloads/tpch/queries/tpch-q11.test index 71409143a..553edf92c 100644 --- a/testdata/workloads/tpch/queries/tpch-q11.test +++ b/testdata/workloads/tpch/queries/tpch-q11.test @@ -14,8 +14,7 @@ group by ps_partkey RESET tpch.q11_sum_tmp RELOAD tpch.q11_sum_tmp ---- RESULTS ----- PARTITIONS -/: 29818 +: 29818 ==== ---- QUERY : TPCH-Q11_QUERY_2 insert overwrite table tpch.q11_sum_tmp @@ -25,6 +24,5 @@ from tpch.q11_part_tmp RESET tpch.q11_part_tmp RELOAD tpch.q11_part_tmp ---- RESULTS ----- PARTITIONS -/: 1 +: 1 ==== diff --git a/testdata/workloads/tpch/queries/tpch-q15.test b/testdata/workloads/tpch/queries/tpch-q15.test index f12edf93e..7210f2f85 100644 --- a/testdata/workloads/tpch/queries/tpch-q15.test +++ b/testdata/workloads/tpch/queries/tpch-q15.test @@ -12,8 +12,7 @@ group by l_suppkey RESET tpch.revenue RELOAD tpch.revenue ---- RESULTS ----- PARTITIONS -/: 10000 +: 10000 ==== ---- QUERY : TPCH-Q15_QUERY_2 insert overwrite table tpch.max_revenue @@ -23,8 +22,7 @@ from tpch.revenue RESET tpch.max_revenue RELOAD tpch.max_revenue ---- RESULTS ----- PARTITIONS -/: 1 +: 1 ==== ---- QUERY : TPCH-Q15_QUERY_3 # Modifications - Added limit diff --git a/testdata/workloads/tpch/queries/tpch-q16.test b/testdata/workloads/tpch/queries/tpch-q16.test index ae4eca0ea..51dd75e5d 100644 --- a/testdata/workloads/tpch/queries/tpch-q16.test +++ b/testdata/workloads/tpch/queries/tpch-q16.test @@ -9,8 +9,7 @@ where not s_comment like '%Customer%Complaints%' RESET tpch.supplier_tmp RELOAD tpch.supplier_tmp ---- RESULTS ----- PARTITIONS -/: 9996 +: 9996 ==== ---- QUERY : TPCH-Q16_QUERY_2 insert overwrite table tpch.q16_tmp @@ -26,8 +25,7 @@ join tpch.supplier_tmp s RESET tpch.q16_tmp RELOAD tpch.q16_tmp ---- RESULTS ----- PARTITIONS -/: 741971 +: 741971 ==== ---- QUERY : TPCH-Q16_QUERY_3 # Modifications: Added limit diff --git a/testdata/workloads/tpch/queries/tpch-q17.test b/testdata/workloads/tpch/queries/tpch-q17.test index 621b6781f..b6280e0bd 100644 --- a/testdata/workloads/tpch/queries/tpch-q17.test +++ b/testdata/workloads/tpch/queries/tpch-q17.test @@ -9,8 +9,7 @@ group by l_partkey RESET tpch.lineitem_tmp RELOAD tpch.lineitem_tmp ---- RESULTS ----- PARTITIONS -/: 200000 +: 200000 ==== ---- QUERY : TPCH-Q17_QUERY_2 # Modifications: Converted selects from multiple tables to joins, diff --git a/testdata/workloads/tpch/queries/tpch-q18.test b/testdata/workloads/tpch/queries/tpch-q18.test index d4568016e..78e0c1695 100644 --- a/testdata/workloads/tpch/queries/tpch-q18.test +++ b/testdata/workloads/tpch/queries/tpch-q18.test @@ -9,8 +9,7 @@ group by l_orderkey RESET q18_tmp RELOAD q18_tmp ---- RESULTS ----- PARTITIONS -/: 1500000 +: 1500000 ==== ---- QUERY : TPCH-Q18_QUERY_2 # Modifications: Cast o_totalprice column due to float/double values not diff --git a/testdata/workloads/tpch/queries/tpch-q2.test b/testdata/workloads/tpch/queries/tpch-q2.test index 36c8fa98a..982001430 100644 --- a/testdata/workloads/tpch/queries/tpch-q2.test +++ b/testdata/workloads/tpch/queries/tpch-q2.test @@ -25,8 +25,7 @@ from partsupp ps RESET tpch.q2_minimum_cost_supplier_tmp1 RELOAD tpch.q2_minimum_cost_supplier_tmp1 ---- RESULTS ----- PARTITIONS -/: 642 +: 642 ==== ---- QUERY : TPCH-Q2_QUERY_2 insert overwrite table tpch.q2_minimum_cost_supplier_tmp2 @@ -39,8 +38,7 @@ group by p_partkey RESET tpch.q2_minimum_cost_supplier_tmp2 RELOAD tpch.q2_minimum_cost_supplier_tmp2 ---- RESULTS ----- PARTITIONS -/: 460 +: 460 ==== ---- QUERY : TPCH-Q2_QUERY_3 # Modifications: Fully qualified name of p_partkey column in the ORDER BY clause diff --git a/testdata/workloads/tpch/queries/tpch-q20.test b/testdata/workloads/tpch/queries/tpch-q20.test index e9abf47e6..92c9b3cad 100644 --- a/testdata/workloads/tpch/queries/tpch-q20.test +++ b/testdata/workloads/tpch/queries/tpch-q20.test @@ -9,8 +9,7 @@ where p_name like 'forest%' RESET tpch.q20_tmp1 RELOAD tpch.q20_tmp1 ---- RESULTS ----- PARTITIONS -/: 2127 +: 2127 ==== ---- QUERY : TPCH-Q20_QUERY_2 insert overwrite table tpch.q20_tmp2 @@ -29,8 +28,7 @@ group by RESET tpch.q20_tmp2 RELOAD tpch.q20_tmp2 ---- RESULTS ----- PARTITIONS -/: 543210 +: 543210 ==== ---- QUERY : TPCH-Q20_QUERY_3 insert overwrite table tpch.q20_tmp3 @@ -47,8 +45,7 @@ from partsupp ps RESET tpch.q20_tmp3 RELOAD tpch.q20_tmp3 ---- RESULTS ----- PARTITIONS -/: 5843 +: 5843 ==== ---- QUERY : TPCH-Q20_QUERY_4 # Modified to use subquery to work around IMP-127 @@ -61,8 +58,7 @@ group by ps_suppkey RESET tpch.q20_tmp4 RELOAD tpch.q20_tmp4 ---- RESULTS ----- PARTITIONS -/: 4397 +: 4397 ==== ---- QUERY : TPCH-Q20_QUERY_5 # Modifications: Added limit diff --git a/testdata/workloads/tpch/queries/tpch-q22.test b/testdata/workloads/tpch/queries/tpch-q22.test index 5c409f60f..e994823cf 100644 --- a/testdata/workloads/tpch/queries/tpch-q22.test +++ b/testdata/workloads/tpch/queries/tpch-q22.test @@ -25,8 +25,7 @@ group by RESET tpch.q22_customer_tmp1 RELOAD tpch.q22_customer_tmp1 ---- RESULTS ----- PARTITIONS -/: 1 +: 1 ==== ---- QUERY : TPCH-Q22_QUERY_2 # Modifications: Updated to use LEFT OUTER JOIN instead of NOT EXISTS, diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py index 28e6d6c70..fdb823344 100644 --- a/tests/beeswax/impala_beeswax.py +++ b/tests/beeswax/impala_beeswax.py @@ -222,19 +222,17 @@ class ImpalaBeeswaxClient(object): self.__do_rpc(lambda: self.imp_service.close(handle)) # The query executed successfully and all the data was fetched. - exec_result = QueryResult(success=True, - data = result_rows, - schema = schema) + exec_result = QueryResult(success=True, data=result_rows, schema=schema) exec_result.summary = 'Returned %d rows' % (len(result_rows)) return exec_result def __fetch_insert_results(self, handle): """Executes an insert query""" result = self.__do_rpc(lambda: self.imp_service.CloseInsert(handle)) - # The insert was successfull + # The insert was successful num_rows = sum(map(int, result.rows_appended.values())) - exec_result = QueryResult(success=True, - data = num_rows) + data = ["%s: %s" % row for row in result.rows_appended.iteritems()] + exec_result = QueryResult(success=True, data=data) exec_result.summary = "Inserted %d rows" % (num_rows,) return exec_result diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py index d6abfa61a..abcdccd4c 100644 --- a/tests/common/impala_test_suite.py +++ b/tests/common/impala_test_suite.py @@ -234,11 +234,7 @@ class ImpalaTestSuite(BaseTestSuite): self.hive_client.create_table(table) def __update_results(self, test_file_name, test_section, exec_result): - if 'PARTITIONS' in test_section: - test_section['PARTITIONS'] = '\n'.join(parse_result_rows(exec_result)) - else: - test_section['RESULTS'] = '\n'.join(parse_result_rows(exec_result)) - + test_section['RESULTS'] = '\n'.join(parse_result_rows(exec_result)) if 'TYPES' in test_section: col_types = [fs.type.upper() for fs in exec_result.schema.fieldSchemas] test_section['TYPES'] = ', '.join(col_types) diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py index 5217e96d0..9200b1680 100644 --- a/tests/common/test_result_verifier.py +++ b/tests/common/test_result_verifier.py @@ -158,29 +158,11 @@ def verify_raw_results(test_section, exec_result): Accepts a raw exec_result object and verifies it matches the expected results This process includes the parsing/transformation of the raw data results into the - result format used in the tests. It also chooses the appropriate result section to - verify from the test case because insert tests need to verify different things - than select tests. + result format used in the tests. """ expected_results = None - # If there is a 'PARTITIONS' section then assume this is an insert test and get the - # expected results from there - if 'PARTITIONS' in test_section: - partition_results = list() - for row in test_section['PARTITIONS'].split('\n'): - # TODO: Currently we don't get the partition names back so strip these out of the - # expected results - if ':' in row: - partition_results.append(row.split(':')[1].strip()) - elif row.strip(): - partition_results.append(row.strip()) - - # TODO: Currently we just return the total number of rows inserted rather than - # rows per partition. This should be updated to validated on a per-partition - # basis. - expected_results = str(sum([int(p_count) for p_count in partition_results])) - elif 'RESULTS' in test_section: + if 'RESULTS' in test_section: expected_results = remove_comments(test_section['RESULTS']) else: LOG.info("No results found. Skipping verification"); diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py index 32a1bdae3..7387db15c 100644 --- a/tests/util/test_file_parser.py +++ b/tests/util/test_file_parser.py @@ -60,7 +60,7 @@ def parse_query_test_file(file_name): """ # Update the valid section names as we support other test types # (ex. planner, data error) - VALID_SECTION_NAMES = ['QUERY', 'RESULTS', 'TYPES', 'PARTITIONS', 'SETUP'] + VALID_SECTION_NAMES = ['QUERY', 'RESULTS', 'TYPES', 'SETUP'] return parse_test_file(file_name, VALID_SECTION_NAMES) def parse_table_constraints(constraints_file):