From 57fcbf7a28339e55a344dece10d87d16631939fa Mon Sep 17 00:00:00 2001 From: Jim Apple Date: Tue, 20 Sep 2016 13:54:55 -0700 Subject: [PATCH] IMPALA-4171: Remove JAR from repo. By ASF rules, we can't have JARs in releases. The releases are just tarballs of the repo. This patch removes from the repo the single JAR there, which was a version of a JAR that is built during data load, with one string changed. The JAR is used only for testing. Instead of building that jar with the different string and saving the result in git, daa loading will now build the jar twice, with one Java source file slightly changed. Change-Id: Icee7b8c32b08e064dea4a14624acff6021ef5ce1 Reviewed-on: http://gerrit.cloudera.org:8080/4499 Reviewed-by: Alex Behm Tested-by: Internal Jenkins --- .gitignore | 3 + bin/check-rat-report.py | 3 +- testdata/bin/copy-udfs-udas.sh | 62 ++++++++++-------- testdata/udfs/README | 8 +-- testdata/udfs/impala-hive-udfs.jar | Bin 3801 -> 0 bytes .../com/cloudera/impala/TestUpdateUdf.java | 12 +--- 6 files changed, 46 insertions(+), 42 deletions(-) delete mode 100644 testdata/udfs/impala-hive-udfs.jar diff --git a/.gitignore b/.gitignore index ec524488c..68a43cba2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ toolchain/ core core.* hs_err_pid*.log + +# Binaries disallowed by ASF rules +*.jar \ No newline at end of file diff --git a/bin/check-rat-report.py b/bin/check-rat-report.py index 06c822dad..a2f3b523b 100755 --- a/bin/check-rat-report.py +++ b/bin/check-rat-report.py @@ -67,7 +67,8 @@ for r in resources: break if not excluded: typename = r.findall('type')[0].attrib['name'] - if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary']): + if not (clean_name[0:9] == 'testdata/' and typename in ['archive', 'binary'] + and clean_name[-4:] <> '.jar'): sys.stderr.write( "%s: %s\n" % ('UNAPPROVED' if approvals else "NO APPROVALS; " + typename, clean_name)) diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh index 4440db112..57c0bc798 100755 --- a/testdata/bin/copy-udfs-udas.sh +++ b/testdata/bin/copy-udfs-udas.sh @@ -26,7 +26,7 @@ if [ x${JAVA_HOME} == x ]; then echo JAVA_HOME not set exit 1 fi -. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1 +. "${IMPALA_HOME}/bin/impala-config.sh" > /dev/null 2>&1 BUILD=0 @@ -47,11 +47,19 @@ done if [ $BUILD -eq 1 ] then - pushd $IMPALA_HOME + pushd "${IMPALA_HOME}" "${MAKE_CMD:-make}" "-j${IMPALA_BUILD_THREADS:-4}" \ TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir - cd $IMPALA_HOME/tests/test-hive-udfs - ${IMPALA_HOME}/bin/mvn-quiet.sh package + cd "${IMPALA_HOME}/tests/test-hive-udfs" + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" + # Change one of the Java files to make a new jar for testing + # purposes, then change it back + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'Old UDF'/'New UDF'/g '{}'" \; + "${IMPALA_HOME}/bin/mvn-quiet.sh" package + find . -type f -name 'TestUpdateUdf.java' -execdir \ + bash -c "sed -i s/'New UDF'/'Old UDF'/g '{}'" \; popd fi @@ -63,27 +71,27 @@ fi # impala-hive-udfs.jar # test-udfs.ll # udf/uda samples (.so/.ll) -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO -hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/udf_test -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so\ - ${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so -hadoop fs -put -f ${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar -hadoop fs -put -f ${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar\ - ${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so\ - ${FILESYSTEM_PREFIX}/test-warehouse -hadoop fs -put -f ${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll\ - ${FILESYSTEM_PREFIX}/test-warehouse +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdas.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/libTestUdfs.SO" +hadoop fs -mkdir -p "${FILESYSTEM_PREFIX}/test-warehouse/udf_test" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse/udf_test/libTestUdfs.so" +hadoop fs -put -f "${HIVE_HOME}/lib/hive-exec-${IMPALA_HIVE_VERSION}.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/hive-exec.jar" +hadoop fs -put -f "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar"\ + "${FILESYSTEM_PREFIX}/test-warehouse/impala-hive-udfs.jar" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/udf-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/libudasample.so"\ + "${FILESYSTEM_PREFIX}/test-warehouse" +hadoop fs -put -f "${IMPALA_HOME}/be/build/latest/udf_samples/uda-sample.ll"\ + "${FILESYSTEM_PREFIX}/test-warehouse" echo "Done copying udf/uda libraries." diff --git a/testdata/udfs/README b/testdata/udfs/README index d8817bb83..31f03b2cf 100644 --- a/testdata/udfs/README +++ b/testdata/udfs/README @@ -1,7 +1,5 @@ -Most of the test UDFs are generated as part of the build. This folder -contains prebuilt UDFs to facilitate tests that verify the UDF binary -can be upgraded. - -The jar is generated from $IMPALA_HOME/tests/test-hive-udfs. +Most of the test UDFs are generated as part of the build. After +copy-udfs-udas.sh is run, This folder contains prebuilt UDFs to +facilitate tests that verify the UDF binary can be upgraded. This is used by test_udfs.py. diff --git a/testdata/udfs/impala-hive-udfs.jar b/testdata/udfs/impala-hive-udfs.jar deleted file mode 100644 index d916937ca8506b7c9e5e6f6ab81571c411d26b3e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3801 zcmbVO2{@GN7aw$GX_yS9q+}1HF*LR?wisi}l2q0)mKwqg#&Sd9)=)y>mU4xrn(Wz2 zmfUP(Ns6*%n6if?-1vWUg>LkJ?sLyP-!jj4&hI_%d*1i_e&!gqEnFa`ZO7{A{r&Ro z#tA%4^sTgEXwxGH&A+*EfFOXIQ5)3<2Ye0&F8=S`OtekWNAxYNn=e7p zLLda%XfXqPF;XNLRSpEs2M$Bdk+W@#kagaQ8W%`VOM7{HdC+QFAP_c4=^xo|0n);r z^;}DQqX+n#2Nr+U&jsh}^qnhnj=L>FtDHk62liqkR-@3=0Uv!U_UO z{UpK)M<7~zyEqYX)-JB{SiBQ~5E(}^tw!-jwnXaW-H~3(hi|{keoAF`k5jk%@k3?( zQa-AW#Dp!?3i+$C!(=_^zRKSarn38YV^eB12$LE!G+NW*^G2aCODnm-qsY8tm1U9J zkjD5MouY#WT}0>@ouXrtd@-^^TsrlM;U&AkL87N;Zs*A2PO5fKP8&+jKXiX~dk^1d zPXll=of;=A+B_&SsOjdZ5Tl;fh;8$k6pA=6NC?fatTxT^qTXvE)d`!yW6O;qjb(my zN3hHUwz_4h#A`o2I(dhja<1P6hpH#4$$uC#mi3?sn`pOqynUfC&)N@S z=gGSlP<+Wl?XXgh_fDuA zG&T@u1gP}>|5uOIN<2yQv=p-Df4;nvZ%1)n-tm(dz2OUoAPv(Q+EouAY#DLKpzbVG zyLiw1WS4ea%}d^axqD~3cio!GZ2pqAr^R*x?m8rCYhSl*j)7lV?kg?jb(?euU0zTR z3nZyY=4J1vU^yIo6JF=0iCr=ADpKMn#tg71>w_JJOCmhI_MRqX&=D03h7edn*v6dU zYEYAXg-U9oLBYXl+1S*ZY~a2T!S{|S7bt3yA8$oX9y$yav@JYY6yV^fqERRdIT`RE z5tZ*={BYcKrq#ER;dJ44jKD2&SLY$So7j0cqhlN!e#yDkWB2u__WLKYJvwPcUGU5K zeq-ZSsW}`mC!}1{u|b2KraOg;U6VZvEQ({wZ2eM-oYeokL06J1nC&doIFe%wmV`p? zR`zJuwlp`|7n{k^JAJ9Ibov+C3W9jjlk6)mj;ZW%G|w!f2OT}q;R#8O_NtdnJWcHg z7TihH}kZT-SBn^lGR3~WcQ>N+m5 z?(HCcbk&#$mMhn580Kh96i{t^g)zG>^Tkx8W9Dr-3!@aTd>{2Y&Hvi{r5v>5uyaxB zoyo{$+{`oMVr1wW7{><*3jNr_$Fw2M{g{QZxIkw~Z)mam{jCR{a3-s`=<4^|XDrk; z%cvjtihsoS=d1VlEKcLkRFqtHLFbN=uHul~g^L|p=6<80)3&{ifeF35+ifDAz&@)z zu(G4bgd^>9y9G+p4vVsKx6R2}4&zcsrn0pHF0#z&*W}ut5Wq?$QK&~~)91OLYmOIu z((F^0K2CQuVVB@BLUax|V%UG-;X>3At|n<;xv*zz`ZbZ{9p?~!s6o<#2+Hl?v0eVd z#ZL_akV_cU$5)ikNd4%S+#~dM&1Ws2oHzpR4XF#mFAE|_4mS)V%D;pKg)7|^|0C=j zNN7Q>ylotJt|~XSlM10#axhrm6k7h4F-7rc5M!N)Aq%-pT;zo{*@;iY#|NAF zx>u*{598^$#N1ufUM3zvECLRyUr}$udx*MBJ7D!@EUt>yA zWlN*u%QJRQ3NWtBlN6imq`XY)znc4MSzMlzA@oiAz?ynQCho@HyxxuLAmu2|dG1rL z?%w(gCR4f(Ga)52p+d0K?Byivn(iSlR4QKM;pv=QH?o? zoqR}_O$pQLU+WbqQ;?^25+SaNR`}FsP#-r=LbvH6$6VuLV_ycxr&yiPU@309E0-~U zxFIX6SsK>GCGI3ZSA}FI5fxI~VqJ{?OlJ=)Qn;g^^ZHEwQoqmSbGqdM*5B_!q(pS2 zQX-lsJwdvftVpl1AuGyW3(+nA0CC3VO9}?>shX9pl}F!4NKc@I@K+a3=>#}S@f5y% zkbT#-!5U3U8`-8LcIV;oD!$$EJavx*Vq;F8Isr#E*K!T01uJomXYTPW9MbX=uBn#r z{*a14Fo^ioSuL&Sg_)==XQ=t&A7<%V0*+$vD^yOEnmSz=IY!X>3=g#4nj!eK@7!fA zr|8#kut_r_Q-+TbGRHkgHg=1iq0THe&P30L_=GB#R{Y-l^ockT)QjjZ**ZcANI2)W z@55)k#hbq2+hnfpGVJv%0Bb0R-*$5kRpF#Ls$U6oZ@IzFm)@2cuiJjOf;tG%WZh9UFkyF&UqLXd%J$en7` z+_{1P*}bi8I)_{R>zoq5V0+Hw+GO>X&D!Z^#^f3WR1~L~9u0r3)Od*zUZ+yTF2fHK?QKibFB1Az&mza)G7Wup zz}W}b#YEsj{pa26?R(Z6=S%d!5x`9*T-9(0>fP5A=RZcH^d2~-ojEGaG0TcYbf9h? zm}fhCHZhjhK7d{shfoh$_P*P|QfAAlDcz4wSB+(;jc{uw*_=^C4JAL0sFwz(N)MHk z*EeL3N!Au#pI+1!VyPXo$uIR`TwBb^>E2l^`Gw6K!@|nX_H)$?%n2|EXk455Tx&J! zVVkRHrZdwr!kOxEE$ll@y^gR^&N2n>ia|hMwJU(v75TfK>W>ncDS%gu8PN~C{zKz_ zKiFo1cs-OG5WW8hU2BS~gKsX7R}es;N8uVjLD)ntucx)S%3BAe0UdL*wEhNN)qLx4 z8(TM%C{_#tT3+sA|E?&jw6QwDjloRfSPy33x*-@yZ4(;#IUtir)&ULv9pDC9S;gO+ zSXM`P6rS{-@W0W_`l4=r;@5}40jFqfr>`vPX6y39G?>GzpMP`AFF