IMPALA-9240: add HTTP code handling to THttpClient.

Before this change Impala Shell is not checking HTTP return codes when
using the hs2-http protocol. The shell is sending a request message
(e.g. send_CloseOperation) but the HTTP call to send this message may
fail. This will result in a failure when reading the reply (e.g. in
recv_CloseOperation) as there is no reply data to read. This will
typically result in an 'EOFError'.

In code that overrides THttpClient.flush(), check the HTTP code that is
returned after the HTTP call is made. If the code is not 1XX
(informational response) or 2XX (successful) then throw an RPCException.

This change does not contain any attempt to recover from an HTTP failures
but it does allow the failure to be detected and a message to be
printed.

In future it may be possible to retry after certain HTTP errors.

Testing:
- Add a new test for impala-shell that tries to connect to an HTTP
  server that always returns a 503 error. Check that an appropriate
  error message is printed.

Change-Id: I3c105f4b8237b87695324d759ffff81821c08c43
Reviewed-on: http://gerrit.cloudera.org:8080/14924
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

shell/impala_client.py

@@ -129,6 +129,17 @@ class MissingThriftMethodException(Exception):
       return self.value
 
 
+class CodeCheckingHttpClient(THttpClient):
+  """Add HTTP response code handling to THttpClient."""
+  def flush(self):
+    THttpClient.flush(self)
+    # At this point the http call has completed.
+    if self.code >= 300:
+      # Report any http response code that is not 1XX (informational response) or
+      # 2XX (successful).
+      raise RPCException("HTTP code {}: {}".format(self.code, self.message))
+
+
 def print_to_stderr(message):
   print >> sys.stderr, message
 
@@ -394,10 +405,11 @@ class ImpalaClient(object):
       else:
         ssl_ctx.check_hostname = False  # Mandated by the SSL lib for CERT_NONE mode.
         ssl_ctx.verify_mode = ssl.CERT_NONE
-      transport = THttpClient(
+      transport = CodeCheckingHttpClient(
           "https://{0}/{1}".format(host_and_port, self.http_path), ssl_context=ssl_ctx)
     else:
-      transport = THttpClient("http://{0}/{1}".format(host_and_port, self.http_path))
+      transport = CodeCheckingHttpClient("http://{0}/{1}".
+          format(host_and_port, self.http_path))
 
     if self.use_ldap:
       # Set the BASIC auth header

tests/shell/test_shell_interactive.py

@@ -18,6 +18,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import httplib
 import logging
 import os
 import pexpect
@@ -26,7 +27,9 @@ import re
 import signal
 import socket
 import sys
+import threading
 from time import sleep
+from contextlib import closing
 
 # This import is the actual ImpalaShell class from impala_shell.py.
 # We rename it to ImpalaShellClass here because we later import another
@@ -40,7 +43,9 @@ from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.skip import SkipIfLocal
 from tests.common.test_dimensions import create_client_protocol_dimension
 from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd,
-                  get_open_sessions_metric)
+                  get_open_sessions_metric, IMPALA_SHELL_EXECUTABLE)
+import SimpleHTTPServer
+import SocketServer
 
 QUERY_FILE_PATH = os.path.join(os.environ['IMPALA_HOME'], 'tests', 'shell')
 
@@ -70,6 +75,20 @@ def tmp_history_file(request):
   return tmp.name
 
 
+class UnavailableRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+  """An HTTP server that always returns 503"""
+  def do_POST(self):
+    self.send_response(code=httplib.SERVICE_UNAVAILABLE, message="Service Unavailable")
+
+
+def get_unused_port():
+  """ Find an unused port http://stackoverflow.com/questions/1365265 """
+  with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+    s.bind(('', 0))
+    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    return s.getsockname()[1]
+
+
 class TestImpalaShellInteractive(ImpalaTestSuite):
   """Test the impala shell interactively"""
 
@@ -842,6 +861,34 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
       result = p.get_result()
       assert "Fetched 0 row" in result.stderr
 
+  def test_http_codes(self, vector):
+    """Check that the shell prints a good message when using hs2-http protocol
+    and the http server returns a 503 error."""
+    protocol = vector.get_value("protocol")
+    if protocol != 'hs2-http':
+      pytest.skip()
+
+    # Start an http server that always returns 503.
+    HOST = "localhost"
+    PORT = get_unused_port()
+    httpd = None
+    http_server_thread = None
+    try:
+      httpd = SocketServer.TCPServer((HOST, PORT), UnavailableRequestHandler)
+      http_server_thread = threading.Thread(target=httpd.serve_forever)
+      http_server_thread.start()
+
+      # Check that we get a message about the 503 error when we try to connect.
+      shell_args = ["--protocol={0}".format(protocol), "-i{0}:{1}".format(HOST, PORT)]
+      shell_proc = pexpect.spawn(IMPALA_SHELL_EXECUTABLE, shell_args)
+      shell_proc.expect("HTTP code 503", timeout=10)
+    finally:
+      # Clean up.
+      if httpd is not None:
+        httpd.shutdown()
+      if http_server_thread is not None:
+        http_server_thread.join()
+
 
 def run_impala_shell_interactive(vector, input_lines, shell_args=None,
                                  wait_until_connected=True):