IMPALA-13015: Dataload fails due to concurrency issue with test.jceks

Move 'hadoop credential' command used for creating test.jceks to
testdata/bin/create-load-data.sh. Earlier it was in bin/load-data.py
which is called in parallel and was causing failures due to race
conditions.

Testing:
- Ran JniFrontendTest#testGetSecretFromKeyStore after data loading and
test ran clean.

Change-Id: I7fbeffc19f2b78c19fee9acf7f96466c8f4f9bcd
Reviewed-on: http://gerrit.cloudera.org:8080/21346
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
This commit is contained in:
Abhishek Rawat
2024-04-22 16:29:48 -07:00
committed by Impala Public Jenkins
parent ec73b542c5
commit f620e5d5c0
2 changed files with 10 additions and 15 deletions

View File

@@ -87,7 +87,6 @@ WORKLOAD_DIR = options.workload_dir
DATASET_DIR = options.dataset_dir
TESTDATA_BIN_DIR = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin')
AVRO_SCHEMA_DIR = "avro_schemas"
TESTDATA_JCEKS_DIR = os.path.join(os.environ['IMPALA_HOME'], 'testdata/jceks')
GENERATE_SCHEMA_CMD = "generate-schema-statements.py --exploration_strategy=%s "\
"--workload=%s --scale_factor=%s --verbose"
@@ -300,14 +299,6 @@ def hive_exec_query_files_parallel(thread_pool, query_files, step_name):
exec_query_files_parallel(thread_pool, query_files, 'hive', step_name)
def exec_hadoop_credential_cmd(secret_key, secret, provider_path, exit_on_error=True):
cmd = ("%s credential create %s -value %s -provider %s"
% (HADOOP_CMD, secret_key, secret, provider_path))
LOG.info("Executing Hadoop command: " + cmd)
exec_cmd(cmd, error_msg="Error executing Hadoop command, exiting",
exit_on_error=exit_on_error)
def main():
logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%H:%M:%S')
LOG.setLevel(logging.DEBUG)
@@ -317,12 +308,6 @@ def main():
#
LOG.debug(' '.join(sys.argv))
jceks_path = TESTDATA_JCEKS_DIR + "/test.jceks"
if os.path.exists(jceks_path):
os.remove(jceks_path)
exec_hadoop_credential_cmd("openai-api-key-secret", "secret",
"localjceks://file" + jceks_path)
all_workloads = available_workloads(WORKLOAD_DIR)
workloads = []
if options.workloads is None: