jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-13600: Fix Flaky JWT Test

Browse Source 
The custom cluster tests for Impala shell JWT authentication all
contain magic numbers for the expected count of RPCs for the hs2-http
protocol. Thus, any time the rpcs are modified, these tests have the
potential to fail.

Since the JWT tests are focused on all JWT authentications either
succeeding or failing, the actual number of rpcs is not relevant. The
tests now use existing metrics to determine the expected rpc count.
Additionally, the tests use existing metrics to determine when the
assertions can run instead of relying on a sleep statement.

The modified tests passed locally and in Jenkins.

Change-Id: Icf0eebd74e1ce10ad24055b7fab4b1901ce61e03
Reviewed-on: http://gerrit.cloudera.org:8080/22201
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
jasonmfehr
2024-12-11 12:14:38 -08:00
committed by Impala Public Jenkins
parent b49f45eacb
commit f5bb759c67
1 changed files with 41 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
tests/custom_cluster/test_shell_jwt_auth.py
View File

@@ -21,7 +21,6 @@ import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.test_dimensions import create_client_protocol_http_transport
from time import sleep
from tests.shell.util import run_impala_shell_cmd
@@ -48,6 +47,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
"-jwt_token_auth=true -jwt_allow_without_tls=true "
.format(JWKS_JSON_PATH))
# Name of the Impala metric containing the total count of hs2-http connections opened.
HS2_HTTP_CONNS = "impala.thrift-server.hiveserver2-http-frontend.total-connections"
@classmethod
def get_workload(self):
return 'functional-query'
@@ -63,19 +65,25 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_success}",
tmp_dir_placeholders=["jwt_auth_success"],
disable_log_buffering=True)
disable_log_buffering=True,
cluster_size=1)
def test_jwt_auth_valid(self, vector):
"""Asserts the Impala shell can authenticate to Impala using JWT authentication.
Also executes a query to ensure the authentication was successful."""
before_rpc_count = self.__get_rpc_count()
# Run a query and wait for it to complete.
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_SIGNED_PATH),
'-q', 'select version()', '--auth_creds_ok_in_clear']
result = run_impala_shell_cmd(vector, args)
self.cluster.get_first_impalad().service.wait_for_metric_value(
"impala-server.backend-num-queries-executed", 1, timeout=15)
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the coordinator
sleep(5)
self.__assert_success_fail_metric(success_count_min=15, success_count_max=16)
query_rpc_count = self.__get_rpc_count() - before_rpc_count
self.__assert_success_fail_metric(success_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -99,10 +107,13 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_fail}",
tmp_dir_placeholders=["jwt_auth_fail"],
disable_log_buffering=True)
disable_log_buffering=True,
cluster_size=1)
def test_jwt_auth_expired(self, vector):
"""Asserts the Impala shell fails to authenticate when it presents a JWT that has a
valid signature but is expired."""
before_rpc_count = self.__get_rpc_count()
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_EXPIRED_PATH),
'-q', 'select version()', '--auth_creds_ok_in_clear']
@@ -110,8 +121,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the coordinator
sleep(5)
self.__assert_success_fail_metric(failure_count_min=4, failure_count_max=4)
self.__wait_for_rpc_count(before_rpc_count + 1)
query_rpc_count = self.__get_rpc_count() - before_rpc_count
self.__assert_success_fail_metric(fail_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -139,10 +151,13 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
impalad_args=IMPALAD_ARGS,
impala_log_dir="{jwt_auth_invalid_jwk}",
tmp_dir_placeholders=["jwt_auth_invalid_jwk"],
disable_log_buffering=True)
disable_log_buffering=True,
cluster_size=1)
def test_jwt_auth_invalid_jwk(self, vector):
"""Asserts the Impala shell fails to authenticate when it presents a JWT that has a
valid signature but is expired."""
before_rpc_count = self.__get_rpc_count()
args = ['--protocol', vector.get_value('protocol'), '-j', '--jwt_cmd',
'cat {0}'.format(TestImpalaShellJWTAuth.JWT_INVALID_JWK),
'-q', 'select version()', '--auth_creds_ok_in_clear']
@@ -150,8 +165,9 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
# Ensure the Impala coordinator is correctly reporting the jwt auth metrics
# must be done before the cluster shuts down since it calls to the coordinator
sleep(5)
self.__assert_success_fail_metric(failure_count_min=4, failure_count_max=4)
self.__wait_for_rpc_count(before_rpc_count + 1)
query_rpc_count = self.__get_rpc_count() - before_rpc_count
self.__assert_success_fail_metric(fail_count=query_rpc_count)
# Shut down cluster to ensure logs flush to disk.
self._stop_impala_cluster()
@@ -174,24 +190,22 @@ class TestImpalaShellJWTAuth(CustomClusterTestSuite):
assert "HTTP code 401: Unauthorized" in result.stderr
assert "Not connected to Impala, could not execute queries." in result.stderr
def __assert_success_fail_metric(self, success_count_min=0, success_count_max=0,
failure_count_min=0, failure_count_max=0):
def __assert_success_fail_metric(self, success_count=0, fail_count=0):
"""Impala emits metrics that count the number of successful and failed JWT
authentications. This function asserts the JWT auth success/fail counters from the
coordinator are within the specified ranges."""
self.__assert_counter(
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-success",
success_count_min, success_count_max)
self.__assert_counter(
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-failure",
failure_count_min, failure_count_max)
coordinator match the expected values."""
actual = self.cluster.get_first_impalad().service.get_metric_values([
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-success",
"impala.thrift-server.hiveserver2-http-frontend.total-jwt-token-auth-failure"])
def __assert_counter(self, counter_name, expected_count_min, expected_count_max):
"""Asserts the value of the specifed counter metric from the coordinator falls
within the specified min and max (inclusive)."""
counter_val = self.cluster.impalads[0].service.get_metric_value(counter_name)
assert actual[0] == success_count, "Expected JWT auth success count to be '{}' but " \
"was '{}'".format(success_count, actual[0])
assert actual[1] == fail_count, "Expected JWT auth failure count to be '{}' but " \
"was '{}'".format(fail_count, actual[1])
assert counter_val >= expected_count_min and counter_val <= expected_count_max, \
"expected counter '{0}' to have a value between '{1}' and '{2}' inclusive " \
"but its value was {3}" \
.format(counter_name, expected_count_min, expected_count_max, counter_val)
def __get_rpc_count(self):
return self.cluster.get_first_impalad().service.get_metric_value(self.HS2_HTTP_CONNS)
def __wait_for_rpc_count(self, expected_count):
self.cluster.get_first_impalad().service.wait_for_metric_value(self.HS2_HTTP_CONNS,
expected_count, allow_greater=True)