mirror of
https://github.com/apache/impala.git
synced 2025-12-19 18:12:08 -05:00
IMPALA-13015: Dataload fails due to concurrency issue with test.jceks
Move 'hadoop credential' command used for creating test.jceks to testdata/bin/create-load-data.sh. Earlier it was in bin/load-data.py which is called in parallel and was causing failures due to race conditions. Testing: - Ran JniFrontendTest#testGetSecretFromKeyStore after data loading and test ran clean. Change-Id: I7fbeffc19f2b78c19fee9acf7f96466c8f4f9bcd Reviewed-on: http://gerrit.cloudera.org:8080/21346 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
committed by
Impala Public Jenkins
parent
ec73b542c5
commit
f620e5d5c0
@@ -87,7 +87,6 @@ WORKLOAD_DIR = options.workload_dir
|
||||
DATASET_DIR = options.dataset_dir
|
||||
TESTDATA_BIN_DIR = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin')
|
||||
AVRO_SCHEMA_DIR = "avro_schemas"
|
||||
TESTDATA_JCEKS_DIR = os.path.join(os.environ['IMPALA_HOME'], 'testdata/jceks')
|
||||
|
||||
GENERATE_SCHEMA_CMD = "generate-schema-statements.py --exploration_strategy=%s "\
|
||||
"--workload=%s --scale_factor=%s --verbose"
|
||||
@@ -300,14 +299,6 @@ def hive_exec_query_files_parallel(thread_pool, query_files, step_name):
|
||||
exec_query_files_parallel(thread_pool, query_files, 'hive', step_name)
|
||||
|
||||
|
||||
def exec_hadoop_credential_cmd(secret_key, secret, provider_path, exit_on_error=True):
|
||||
cmd = ("%s credential create %s -value %s -provider %s"
|
||||
% (HADOOP_CMD, secret_key, secret, provider_path))
|
||||
LOG.info("Executing Hadoop command: " + cmd)
|
||||
exec_cmd(cmd, error_msg="Error executing Hadoop command, exiting",
|
||||
exit_on_error=exit_on_error)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%H:%M:%S')
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
@@ -317,12 +308,6 @@ def main():
|
||||
#
|
||||
LOG.debug(' '.join(sys.argv))
|
||||
|
||||
jceks_path = TESTDATA_JCEKS_DIR + "/test.jceks"
|
||||
if os.path.exists(jceks_path):
|
||||
os.remove(jceks_path)
|
||||
exec_hadoop_credential_cmd("openai-api-key-secret", "secret",
|
||||
"localjceks://file" + jceks_path)
|
||||
|
||||
all_workloads = available_workloads(WORKLOAD_DIR)
|
||||
workloads = []
|
||||
if options.workloads is None:
|
||||
|
||||
Reference in New Issue
Block a user