diff --git a/.gitignore b/.gitignore index 704f03d0e..32e35288c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ bin/version.info bin/impala-config-local.sh .cache .cdh +.cdp # distcc options .impala_compiler_opts diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 05a0a8113..7c449029a 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -384,7 +384,7 @@ def download_cdh_components(toolchain_root, cdh_components, url_prefix): cdh_components_home = os.environ.get("CDH_COMPONENTS_HOME") if not cdh_components_home: logging.error("Impala environment not set up correctly, make sure " - "$CDH_COMPONENTS_HOME is present.") + "$CDH_COMPONENTS_HOME is set.") sys.exit(1) # Create the directory where CDH components live if necessary. @@ -417,72 +417,45 @@ def download_cdh_components(toolchain_root, cdh_components, url_prefix): execute_many(download, cdh_components) -def download_ranger(toolchain_root): - env_var_version = "IMPALA_RANGER_VERSION" - version = os.environ.get(env_var_version) - # If Ranger has already been downloaded, do not re-download it. - if not version: - raise Exception("Could not find version for Ranger in environment var {0}" - .format(env_var_version)) - pkg_directory = "{0}/ranger-{1}-admin".format(toolchain_root, version) - if os.path.isdir(pkg_directory): return - file_name = "ranger-{0}-admin.tar.gz".format(version) - download_url = "{0}/ranger/{1}/{2}".format(TOOLCHAIN_HOST, version, file_name) - wget_and_unpack_package(download_url, file_name, toolchain_root, False) - - -def download_cdp_hive(toolchain_root): - use_cdp_hive = os.getenv("USE_CDP_HIVE") == "true" - if not use_cdp_hive: - return - - if "CDP_BUILD_NUMBER" not in os.environ: - logging.error("Impala environment not set up correctly. CDP_BUILD_NUMBER must " - "be set when USE_CDP_HIVE is set to true. Make sure " - "impala-config.sh is sourced.") - sys.exit(1) - - cdp_build_number = os.environ.get("CDP_BUILD_NUMBER") - url_prefix = "https://{0}/build/cdp_components/{1}/tarballs/".format( - cdh_host, cdp_build_number) +def download_cdp_components(cdp_components, url_prefix): + """Downloads and unpacks the CDP components for a given URL prefix into + $CDP_COMPONENTS_HOME if not found.""" cdp_components_home = os.environ.get("CDP_COMPONENTS_HOME") if not cdp_components_home: logging.error("Impala environment not set up correctly, make sure " "$CDP_COMPONENTS_HOME is set.") sys.exit(1) - # Create the directory where CDH components live if necessary. + # Create the directory where CDP components live if necessary. if not os.path.exists(cdp_components_home): os.makedirs(cdp_components_home) - version = os.environ.get("IMPALA_HIVE_VERSION") - # TODO the naming convention of the CDP Hive is different than cdh6.x hive - # The tar balls are named for example like apache-hive-3.1.0.6.0.99.0-9-bin.tar.gz - dir_name = "apache-hive-{0}-bin".format(version) - pkg_directory = os.path.join(cdp_components_home, dir_name) - if os.path.isdir(pkg_directory): - return + def download(component_name): + pkg_directory = "{0}/{1}".format(cdp_components_home, component_name) + if os.path.isdir(pkg_directory): return + file_name = "{0}.tar.gz".format(component_name) + download_path = "{0}/{1}".format(url_prefix, file_name) + wget_and_unpack_package(download_path, file_name, cdp_components_home, False) - # Download the package if it doesn't exist - file_name = "{0}.tar.gz".format(dir_name) - download_path = url_prefix + file_name - wget_and_unpack_package(download_path, file_name, cdp_components_home, False) + execute_many(download, cdp_components) if __name__ == "__main__": """Validates the presence of $IMPALA_HOME and $IMPALA_TOOLCHAIN in the environment.- - By checking $IMPALA_HOME is present, we assume that IMPALA_{LIB}_VERSION will be present + By checking $IMPALA_HOME is set, we assume that IMPALA_{LIB}_VERSION will be set as well. Will create the directory specified by $IMPALA_TOOLCHAIN if it doesn't exist yet. Each of the packages specified in `packages` is downloaded and extracted into $IMPALA_TOOLCHAIN. If $DOWNLOAD_CDH_COMPONENTS is true, the presence of - $CDH_DOWNLOAD_HOST and $CDH_BUILD_NUMBER will be checked and this function will also - download the following CDH components into the directory specified by - $CDH_COMPONENTS_HOME. - - hadoop (downloaded from $CDH_DOWNLOAD_HOST for a given $CDH_BUILD_NUMBER) - - hbase (downloaded from $CDH_DOWNLOAD_HOST for a given $CDH_BUILD_NUMBER) - - hive (downloaded from $CDH_DOWNLOAD_HOST for a given $CDH_BUILD_NUMBER) - - sentry (downloaded from $CDH_DOWNLOAD_HOST for a given $CDH_BUILD_NUMBER) + $IMPALA_TOOLCHAIN_HOST and $CDH_BUILD_NUMBER will be checked and this function will also + download the following CDH/CDP components into the directory specified by + $CDH_COMPONENTS_HOME/$CDP_COMPONENTS_HOME. + - hadoop (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDH_BUILD_NUMBER) + - hbase (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDH_BUILD_NUMBER) + - hive (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDH_BUILD_NUMBER) + - sentry (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDH_BUILD_NUMBER) - llama-minikdc (downloaded from $TOOLCHAIN_HOST) + - ranger (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDP_BUILD_NUMBER) + - hive3 (downloaded from $IMPALA_TOOLCHAIN_HOST for a given $CDP_BUILD_NUMBER) """ logging.basicConfig(level=logging.INFO, format='%(asctime)s %(threadName)s %(levelname)s: %(message)s') @@ -498,7 +471,7 @@ if __name__ == "__main__": toolchain_root = os.environ.get("IMPALA_TOOLCHAIN") if not toolchain_root: logging.error("Impala environment not set up correctly, make sure " - "$IMPALA_TOOLCHAIN is present.") + "$IMPALA_TOOLCHAIN is set.") sys.exit(1) if not os.path.exists(toolchain_root): @@ -527,13 +500,13 @@ if __name__ == "__main__": # Download the CDH components if necessary. if not os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true": sys.exit(0) - if "CDH_DOWNLOAD_HOST" not in os.environ or "CDH_BUILD_NUMBER" not in os.environ: + if "IMPALA_TOOLCHAIN_HOST" not in os.environ or "CDH_BUILD_NUMBER" not in os.environ: logging.error("Impala environment not set up correctly, make sure " "impala-config.sh is sourced.") sys.exit(1) - cdh_host = os.environ.get("CDH_DOWNLOAD_HOST") - cdh_build_number = os.environ.get("CDH_BUILD_NUMBER") + toolchain_host = os.environ["IMPALA_TOOLCHAIN_HOST"] + cdh_build_number = os.environ["CDH_BUILD_NUMBER"] cdh_components = map(Package, ["hadoop", "hbase", "sentry"]) use_cdp_hive = os.getenv("USE_CDP_HIVE") == "true" @@ -546,8 +519,8 @@ if __name__ == "__main__": "to use the toolchain Kudu.") sys.exit(1) cdh_components += [Package("kudu")] - download_path_prefix= \ - "https://{0}/build/cdh_components/{1}/tarballs/".format(cdh_host, + download_path_prefix = \ + "https://{0}/build/cdh_components/{1}/tarballs/".format(toolchain_host, cdh_build_number) download_cdh_components(toolchain_root, cdh_components, download_path_prefix) @@ -558,5 +531,14 @@ if __name__ == "__main__": download_path_prefix = "{0}/cdh_components/".format(TOOLCHAIN_HOST) download_cdh_components(toolchain_root, cdh_components, download_path_prefix) - download_ranger(toolchain_root) - download_cdp_hive(toolchain_root) + cdp_build_number = os.environ["CDP_BUILD_NUMBER"] + cdp_components = [ + "ranger-{0}-admin".format(os.environ.get("IMPALA_RANGER_VERSION")), + ] + if use_cdp_hive: + cdp_components.append("apache-hive-{0}-bin" + .format(os.environ.get("IMPALA_HIVE_VERSION"))) + download_path_prefix = \ + "https://{0}/build/cdp_components/{1}/tarballs".format(toolchain_host, + cdp_build_number) + download_cdp_components(cdp_components, download_path_prefix) diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh index 72dfed298..936ab9ef2 100755 --- a/bin/create-test-configuration.sh +++ b/bin/create-test-configuration.sh @@ -213,6 +213,7 @@ fi popd RANGER_SERVER_CONF_DIR="${RANGER_HOME}/ews/webapp/WEB-INF/classes/conf" +RANGER_SERVER_CONFDIST_DIR="${RANGER_HOME}/ews/webapp/WEB-INF/classes/conf.dist" RANGER_SERVER_LIB_DIR="${RANGER_HOME}/ews/webapp/WEB-INF/lib" if [[ ! -d "${RANGER_SERVER_CONF_DIR}" ]]; then mkdir -p "${RANGER_SERVER_CONF_DIR}" @@ -221,7 +222,7 @@ fi cp -f "${RANGER_TEST_CONF_DIR}/java_home.sh" "${RANGER_SERVER_CONF_DIR}" cp -f "${RANGER_TEST_CONF_DIR}/ranger-admin-env-logdir.sh" "${RANGER_SERVER_CONF_DIR}" cp -f "${RANGER_TEST_CONF_DIR}/ranger-admin-env-piddir.sh" "${RANGER_SERVER_CONF_DIR}" -cp -f "${RANGER_TEST_CONF_DIR}/security-applicationContext.xml" \ +cp -f "${RANGER_SERVER_CONFDIST_DIR}/security-applicationContext.xml" \ "${RANGER_SERVER_CONF_DIR}" if [[ -f "${POSTGRES_JDBC_DRIVER}" ]]; then cp -f "${POSTGRES_JDBC_DRIVER}" "${RANGER_SERVER_LIB_DIR}" diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 75bf89cb3..445c5cc43 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -157,15 +157,15 @@ if [[ $OSTYPE == "darwin"* ]]; then unset IMPALA_OPENSSL_URL fi -: ${CDH_DOWNLOAD_HOST:=native-toolchain.s3.amazonaws.com} -export CDH_DOWNLOAD_HOST +: ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com} +export IMPALA_TOOLCHAIN_HOST export CDH_MAJOR_VERSION=6 export CDH_BUILD_NUMBER=1009254 -export CDP_BUILD_NUMBER=976603 +export CDP_BUILD_NUMBER=1013201 export IMPALA_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT export IMPALA_HBASE_VERSION=2.1.0-cdh6.x-SNAPSHOT export IMPALA_SENTRY_VERSION=2.1.0-cdh6.x-SNAPSHOT -export IMPALA_RANGER_VERSION=1.2.0 +export IMPALA_RANGER_VERSION=1.2.0.6.0.99.0-45 export IMPALA_PARQUET_VERSION=1.9.0-cdh6.x-SNAPSHOT export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0 @@ -173,7 +173,7 @@ export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT export KUDU_JAVA_VERSION=1.10.0-cdh6.x-SNAPSHOT export USE_CDP_HIVE=${USE_CDP_HIVE-false} if $USE_CDP_HIVE; then - export IMPALA_HIVE_VERSION=3.1.0.6.0.99.0-9 + export IMPALA_HIVE_VERSION=3.1.0.6.0.99.0-45 else export IMPALA_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT fi @@ -493,11 +493,8 @@ else export CDH_COMPONENTS_HOME="$IMPALA_HOME/thirdparty" fi -# The directory in which all the CDP components live. Applies only when -# USE_CDP_HIVE is set to true -if $USE_CDP_HIVE; then - export CDP_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdp_components-$CDP_BUILD_NUMBER" -fi +# The directory in which all the thirdparty CDP components live. +export CDP_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdp_components-$CDP_BUILD_NUMBER" # Typically we build against a snapshot build of Hadoop that includes everything we need # for building Impala and running a minicluster. @@ -529,10 +526,9 @@ export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_V export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}" export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources" -export RANGER_HOME="${IMPALA_TOOLCHAIN}/ranger-${IMPALA_RANGER_VERSION}-admin" +export RANGER_HOME="${CDP_COMPONENTS_HOME}/ranger-${IMPALA_RANGER_VERSION}-admin" export RANGER_CONF_DIR="$IMPALA_HOME/fe/src/test/resources" - # Extract the first component of the hive version. export IMPALA_HIVE_MAJOR_VERSION=$(echo "$IMPALA_HIVE_VERSION" | cut -d . -f 1) if $USE_CDP_HIVE; then @@ -745,16 +741,20 @@ echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH" echo "LD_PRELOAD = $LD_PRELOAD" echo "POSTGRES_JDBC_DRIVER = $POSTGRES_JDBC_DRIVER" echo "IMPALA_TOOLCHAIN = $IMPALA_TOOLCHAIN" +echo "METASTORE_DB = $METASTORE_DB" echo "DOWNLOAD_CDH_COMPONENTS = $DOWNLOAD_CDH_COMPONENTS" echo "IMPALA_MAVEN_OPTIONS = $IMPALA_MAVEN_OPTIONS" -echo "CDH_DOWNLOAD_HOST = $CDH_DOWNLOAD_HOST" +echo "IMPALA_TOOLCHAIN_HOST = $IMPALA_TOOLCHAIN_HOST" echo "CDH_BUILD_NUMBER = $CDH_BUILD_NUMBER" +echo "CDH_COMPONENTS_HOME = $CDH_COMPONENTS_HOME" echo "CDP_BUILD_NUMBER = $CDP_BUILD_NUMBER" -if $USE_CDP_HIVE; then - echo "CDP_COMPONENTS_HOME = $CDP_COMPONENTS_HOME" -fi +echo "CDP_COMPONENTS_HOME = $CDP_COMPONENTS_HOME" +echo "IMPALA_HADOOP_VERSION = $IMPALA_HADOOP_VERSION" echo "IMPALA_HIVE_VERSION = $IMPALA_HIVE_VERSION" -echo "METASTORE_DB = $METASTORE_DB" +echo "IMPALA_HBASE_VERSION = $IMPALA_HBASE_VERSION" +echo "IMPALA_SENTRY_VERSION = $IMPALA_SENTRY_VERSION" +echo "IMPALA_KUDU_VERSION = $IMPALA_KUDU_VERSION" +echo "IMPALA_RANGER_VERSION = $IMPALA_RANGER_VERSION" # Kerberos things. If the cluster exists and is kerberized, source # the required environment. This is required for any hadoop tool to diff --git a/bin/mvn-quiet.sh b/bin/mvn-quiet.sh index beeaeb95f..709ad28bf 100755 --- a/bin/mvn-quiet.sh +++ b/bin/mvn-quiet.sh @@ -32,7 +32,8 @@ Directory $(pwd) EOF if ! mvn $IMPALA_MAVEN_OPTIONS "$@" | \ - tee -a "$LOG_FILE" | grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test; then + tee -a "$LOG_FILE" | grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test | \ + grep -v "Could not transfer"; then echo "mvn $IMPALA_MAVEN_OPTIONS $@ exited with code $?" exit 1 fi diff --git a/buildall.sh b/buildall.sh index 1b561310d..45bb776b0 100755 --- a/buildall.sh +++ b/buildall.sh @@ -536,17 +536,20 @@ create_log_dirs bootstrap_dependencies -# Create .cdh file that contains the CDH_BUILD_NUMBER. If the content -# of the file is different than the one in the environment variable, -# append -U into IMPALA_MAVEN_OPTION to force Maven to update its local -# cache. +# Create .cdh and .cdp files that contains the CDH_BUILD_NUMBER and CDP_BUILD_NUMBER +# respectively. If the content of the files are different than the ones in the +# environment variable, append -U into IMPALA_MAVEN_OPTION to force Maven to update its +# local cache. CDH_FILE="${IMPALA_HOME}/.cdh" -if [[ -f ${CDH_FILE} ]]; then - if [[ $(cat ${CDH_FILE}) != ${CDH_BUILD_NUMBER} ]]; then +CDP_FILE="${IMPALA_HOME}/.cdp" +if [[ -f ${CDH_FILE} && -f ${CDP_FILE} ]]; then + if [[ $(cat ${CDH_FILE}) != ${CDH_BUILD_NUMBER} || \ + $(cat ${CDP_FILE}) != ${CDP_BUILD_NUMBER} ]]; then export IMPALA_MAVEN_OPTIONS="${IMPALA_MAVEN_OPTIONS} -U" fi fi echo "${CDH_BUILD_NUMBER}" > ${CDH_FILE} +echo "${CDP_BUILD_NUMBER}" > ${CDP_FILE} if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then build_fe diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java index fbef9cb8e..809368ff9 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java @@ -2880,8 +2880,7 @@ public class AuthorizationStmtTest extends FrontendTestBase { try { createRangerPolicy(policyName, json); - // TODO: replace with the new API in RANGER-2349. - rangerImpalaPlugin_.init(); + rangerImpalaPlugin_.refreshPoliciesAndTags(); // Queries on columns that are not masked should be allowed. authorize("select id from functional.alltypes") @@ -2996,8 +2995,7 @@ public class AuthorizationStmtTest extends FrontendTestBase { try { createRangerPolicy(policyName, json); - // TODO: replace with the new API in RANGER-2349. - rangerImpalaPlugin_.init(); + rangerImpalaPlugin_.refreshPoliciesAndTags(); // Queries on tables that are not filtered should be allowed. authorize("select string_col from functional_kudu.alltypes") @@ -3294,8 +3292,7 @@ public class AuthorizationStmtTest extends FrontendTestBase { } authzManager.grantPrivilege(requests); - // TODO: replace with the new API in RANGER-2349. - rangerImpalaPlugin_.init(); + rangerImpalaPlugin_.refreshPoliciesAndTags(); } /** diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml index 5b4c6fc9a..1f8d87851 100644 --- a/impala-parent/pom.xml +++ b/impala-parent/pom.xml @@ -110,9 +110,17 @@ under the License. false + + impala.cdp.repo + https://${env.IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${env.CDP_BUILD_NUMBER}/maven + Impala CDP Repository + + false + + impala.cdh.repo - https://${env.CDH_DOWNLOAD_HOST}/build/cdh_components/${env.CDH_BUILD_NUMBER}/maven + https://${env.IMPALA_TOOLCHAIN_HOST}/build/cdh_components/${env.CDH_BUILD_NUMBER}/maven Impala CDH Repository true diff --git a/testdata/cluster/ranger/ranger-admin-default-site.xml.template b/testdata/cluster/ranger/ranger-admin-default-site.xml.template index 5386885a9..4a1ed614b 100644 --- a/testdata/cluster/ranger/ranger-admin-default-site.xml.template +++ b/testdata/cluster/ranger/ranger-admin-default-site.xml.template @@ -214,7 +214,7 @@ ranger.unixauth.keystore.password - _ + password ranger.unixauth.truststore @@ -226,7 +226,7 @@ ranger.unixauth.truststore.password - _ + changeit diff --git a/testdata/cluster/ranger/security-applicationContext.xml b/testdata/cluster/ranger/security-applicationContext.xml deleted file mode 100644 index 93c4816bb..000000000 --- a/testdata/cluster/ranger/security-applicationContext.xml +++ /dev/null @@ -1,136 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -