IMPALA-14287: Resolve environment variables in REST server configurations

This change adds a step to REST server configuration loading that
resolves environment variables noted as ${ENV:VARIABLE_NAME} format. If
the environment variable is not set, then the reference text remains the
same and Impala logs an error.

Tests:
  - unit tests added

Change-Id: I3faccc15d012c389703c58371a4d38cca82bef60
Reviewed-on: http://gerrit.cloudera.org:8080/23457
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

fe/src/main/java/org/apache/impala/service/catalogmanager/ConfigLoader.java

@@ -25,16 +25,29 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.Properties;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.impala.common.ImpalaRuntimeException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ConfigLoader {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ConfigLoader.class);
+
   private final File configFolder_;
+  private final UnaryOperator<String> envVarResolver_;
 
   public ConfigLoader(File configFolder) {
+    this(configFolder, System::getenv);
+  }
+
+  public ConfigLoader(File configFolder, UnaryOperator<String> envVarResolver) {
     this.configFolder_ = configFolder;
+    this.envVarResolver_ = envVarResolver;
   }
 
   public List<Properties> loadConfigs() throws ImpalaRuntimeException {
@@ -68,9 +81,40 @@ public class ConfigLoader {
     try (InputStream in = Files.newInputStream(file.toPath())) {
       props.load(in);
     }
+    this.resolveEnvVars(props);
     return props;
   }
 
+  private void resolveEnvVars(Properties props) {
+    for (String key : props.stringPropertyNames()) {
+      String value = props.getProperty(key);
+      String resolved = resolveEnvVar(value);
+      props.setProperty(key, resolved);
+    }
+  }
+
+  private String resolveEnvVar(String input) {
+    if (input == null || input.isEmpty()) return input;
+    Pattern pattern = Pattern.compile("\\$\\{ENV:([a-zA-Z][a-zA-Z0-9_-]*)}");
+    Matcher matcher = pattern.matcher(input);
+    StringBuffer sb = new StringBuffer();
+    while (matcher.find()) {
+      String envVarName = matcher.group(1);
+      String envVarValue = this.envVarResolver_.apply(envVarName);
+      String replacement;
+      if (envVarValue == null) {
+        LOG.error("Unable to resolve environment variable: '{}'", envVarName);
+        replacement = matcher.group(0);
+      }
+      else {
+        replacement = envVarValue;
+      }
+      matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement));
+    }
+    matcher.appendTail(sb);
+    return sb.toString();
+  }
+
   private void checkPropertyValue(String configFile, Properties props, String key,
       String expectedValue) {
     if (!props.containsKey(key)) {

fe/src/test/java/org/apache/impala/service/catalogmanager/ConfigLoaderTest.java

@@ -18,6 +18,7 @@ package org.apache.impala.service.catalogmanager;
 
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -124,4 +125,97 @@ public class ConfigLoaderTest {
     List<Properties> configs = loader.loadConfigs();
     assertTrue(configs.isEmpty());
   }
-}
+
+  @Test
+  public void testEnvVarResolutionResolved() throws Exception {
+    String pathValue = System.getenv("PATH");
+    createConfigFile("resolved.properties",
+        "path=${ENV:PATH}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+    ConfigLoader loader = new ConfigLoader(tempDir);
+    List<Properties> configs = loader.loadConfigs();
+    assertEquals(1, configs.size());
+    assertNotNull(pathValue);
+    assertEquals(pathValue, configs.get(0).getProperty("path"));
+  }
+
+  @Test
+  public void testEnvVarResolutionUnresolved() throws Exception {
+    createConfigFile("unresolved.properties",
+        "secret=${ENV:FAKE_SECRET}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+    ConfigLoader loader = new ConfigLoader(tempDir);
+    List<Properties> configs = loader.loadConfigs();
+    assertEquals(1, configs.size());
+    assertEquals("${ENV:FAKE_SECRET}", configs.get(0).getProperty("secret"));
+  }
+
+  @Test
+  public void testEnvVarInvalidSyntax() throws Exception {
+    createConfigFile("invalid.properties",
+        "invalid=${SOME_VAR}\nconnector.name=iceberg\niceberg.catalog.type=rest\n");
+    ConfigLoader loader = new ConfigLoader(tempDir);
+    List<Properties> configs = loader.loadConfigs();
+    assertEquals(1, configs.size());
+    assertEquals("${SOME_VAR}", configs.get(0).getProperty("invalid"));
+  }
+
+  @Test
+  public void testMultipleEnvVars() throws Exception {
+    String pathValue = System.getenv("PATH");
+    createConfigFile("multiple.properties",
+        "path=${ENV:PATH}/dir ${ENV:FAKE}\n"
+            + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+    ConfigLoader loader = new ConfigLoader(tempDir);
+    List<Properties> configs = loader.loadConfigs();
+    assertEquals(1, configs.size());
+    String resolvedPath = configs.get(0).getProperty("path");
+    String expected = pathValue + "/dir ${ENV:FAKE}";
+    assertEquals(expected, resolvedPath);
+  }
+
+  @Test
+  public void testMultipleValidEnvVars() throws Exception {
+    String pathValue = System.getenv("PATH");
+    String userValue = System.getenv("USER");
+    String shellValue = System.getenv("SHELL");
+
+    createConfigFile("multiple_env_vars_with_specific_names.properties",
+        "path=${ENV:PATH}\n"
+            + "user=${ENV:USER}\n"
+            + "shell=${ENV:SHELL}\n"
+            + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+
+    ConfigLoader loader = new ConfigLoader(tempDir);
+    List<Properties> configs = loader.loadConfigs();
+
+    assertEquals(1, configs.size());
+    Properties config = configs.get(0);
+
+    assertNotNull(pathValue);
+    assertEquals(pathValue, config.getProperty("path"));
+
+    assertNotNull(userValue);
+    assertEquals(userValue, config.getProperty("user"));
+
+    assertNotNull(shellValue);
+    assertEquals(shellValue, config.getProperty("shell"));
+  }
+
+  @Test
+  public void testEnvVarWithNewLines() throws Exception {
+    createConfigFile("newline.properties",
+        "key=${ENV:CUSTOM_VAR}\n"
+            + "connector.name=iceberg\niceberg.catalog.type=rest\n");
+
+    ConfigLoader loader = new ConfigLoader(tempDir, envVar -> {
+      if (envVar.equals("CUSTOM_VAR")) {
+        return "VALUE\nWITH\nNEWLINE";
+      }
+      return null;
+    });
+
+    List<Properties> configs = loader.loadConfigs();
+    assertEquals(1, configs.size());
+    Properties config = configs.get(0);
+    assertEquals("VALUE\nWITH\nNEWLINE", config.getProperty("key"));
+  }
+}