diff --git a/.gitignore b/.gitignore index ec524488c..68a43cba2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ toolchain/ core core.* hs_err_pid*.log + +# Binaries disallowed by ASF rules +*.jar \ No newline at end of file diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py index 06c822dad..a2f3b523b 100755 --- a/bin/check-rat-report.py +++ b/bin/check-rat-report.py @@ -67,7 +67,8 @@ for r in resources: break if not excluded: typename = r.findall('type')[0].attrib['name'] - if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']): + if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary'] + and clean_name[-4:] <> '.jar'): sys.stderr.write( "%s: %s\n" % ('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name)) diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh index 4440db112..57c0bc798 100755 --- a/testdata/bin/copy-udfs-udas.sh +++ b/testdata/bin/copy-udfs-udas.sh @@ -26,7 +26,7 @@ if [ x${JAVA_HOME} == x ]; then echo JAVA_HOME not set exit 1 fi -. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 +. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1 BUILD=0 @@ -47,11 +47,19 @@ done if [ $BUILD -eq 1 ] then - pushd $IMPALA_HOME + pushd "${IMPALA_HOME}" "${MAKE_CMD:-make}" "-j${IMPALA_BUILD_THREADS:-4}" \ TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir - cd $IMPALA_HOME/tests/test-hive-udfs - ${IMPALA_HOME}/bin/mvn-quiet.sh package + cd "${IMPALA_HOME}/tests/test-hive-udfs" + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" + # Change one of the Java files to make a new jar for testing + # purposes, then change it back + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'Old UDF'/'New UDF'/g '{}'" \; + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'New UDF'/'Old UDF'/g '{}'" \; popd fi @@ -63,27 +71,27 @@ fi # impala-hive-udfs.jar # test-udfs.ll # udf/uda samples (.so/.ll) -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO -hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/udf_test -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so -hadoop fs -put -f ${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar -hadoop fs -put -f ${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO" +hadoop fs -mkdir -p "${FILESYSTEM_PREFIX}/test-warehouse/udf_test" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so" +hadoop fs -put -f "${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar" +hadoop fs -put -f "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" echo "Done copying udf/uda libraries." diff --git a/testdata/udfs/README b/testdata/udfs/README index d8817bb83..31f03b2cf 100644 --- a/testdata/udfs/README +++ b/testdata/udfs/README @@ -1,7 +1,5 @@ -Most of the test UDFs are generated as part of the build. This folder -contains prebuilt UDFs to facilitate tests that verify the UDF binary -can be upgraded. - -The jar is generated from $IMPALA_HOME/tests/test-hive-udfs. +Most of the test UDFs are generated as part of the build. After +copy-udfs-udas.sh is run, This folder contains prebuilt UDFs to +facilitate tests that verify the UDF binary can be upgraded. This is used by test_udfs.py. diff --git a/testdata/udfs/impala-hive-udfs.jar b/testdata/udfs/impala-hive-udfs.jar deleted file mode 100644 index d916937ca..000000000 Binary files a/testdata/udfs/impala-hive-udfs.jar and /dev/null differ diff --git a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java index f29e8afad..2d76d73ee 100644 --- a/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java +++ b/tests/test-hive-udfs/src/main/java/com/cloudera/impala/TestUpdateUdf.java @@ -21,8 +21,9 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.io.Text; /** - * Simple UDF for testing updating a UDF in HDFS. This file is built and - * the old version of the jar is saved to $IMPALA_HOME/testdata/udfs/ + * Simple UDF for testing updating a UDF in HDFS. This file is built + * twice, with one version of the jar having TestUpdateUdf.evaluate() + * return "Old UDF" and the other "New UDF". * * The build produces a new version of the UDF. The tests make sure the * jar can be updated without having to restart Impalad. @@ -31,14 +32,7 @@ public class TestUpdateUdf extends UDF { public TestUpdateUdf() { } - public Text evaluate() { - return new Text("New UDF"); - } - - // Use this function instead of the other evaluate to regenerate the checked in jar. - /* public Text evaluate() { return new Text("Old UDF"); } - */ }