backport the commit from TESB-32453 and TESB-32391

* TESB-31465:[7.3.1] Studio ESB build performance issue

* TESB-32391
[7.3.1] Incorrect OSGi manifest for Routes

* TESB-32391
[7.3.1] Incorrect OSGi manifest for Routes

* TESB-32391:[7.3.1] Incorrect OSGi manifest for Routes

* TESB-32391:[7.3.1] Incorrect OSGi manifest for Routes

* TESB-32391:[7.3.1] Incorrect OSGi manifest for Routes

Conflicts:
	main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/scriptsmanager/esb/JobJavaScriptOSGIForESBManager.java

main/plugins/org.talend.repository/src/main/java/org/talend/repository/ui/wizards/exportjob/scriptsmanager/esb/JobJavaScriptOSGIForESBManager.java

@@ -12,11 +12,16 @@
 // ============================================================================
 package org.talend.repository.ui.wizards.exportjob.scriptsmanager.esb;
 
+import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.io.PrintWriter;
 import java.net.MalformedURLException;
 import java.net.URI;
@@ -30,10 +35,9 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.jar.Attributes;
 import java.util.jar.Manifest;
 import java.util.regex.Matcher;
@@ -43,11 +47,15 @@ import java.util.stream.Stream;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.map.MultiKeyMap;
+import org.apache.commons.lang3.StringUtils;
 import org.eclipse.core.resources.IFile;
 import org.eclipse.core.resources.IFolder;
+import org.eclipse.core.runtime.IPath;
 import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.Platform;
 import org.eclipse.core.runtime.Status;
 import org.eclipse.emf.common.util.EList;
+import org.osgi.framework.Bundle;
 import org.talend.commons.exception.ExceptionHandler;
 import org.talend.commons.exception.PersistenceException;
 import org.talend.commons.utils.generation.JavaUtils;
@@ -98,15 +106,12 @@ import org.talend.repository.ui.wizards.exportjob.scriptsmanager.JobJavaScriptsM
 import org.talend.repository.utils.EmfModelUtils;
 import org.talend.repository.utils.TemplateProcessor;
 
-import aQute.bnd.header.Attrs;
+import aQute.bnd.header.Parameters;
 import aQute.bnd.osgi.Analyzer;
 import aQute.bnd.osgi.Clazz;
-import aQute.bnd.osgi.Descriptors;
+import aQute.bnd.osgi.Domain;
 import aQute.bnd.osgi.FileResource;
 import aQute.bnd.osgi.Jar;
-import aQute.bnd.service.AnalyzerPlugin;
-import aQute.bnd.service.Plugin;
-import aQute.service.reporter.Reporter;
 
 /**
  * DOC ycbai class global comment. Detailled comment
@@ -118,13 +123,51 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
 
     protected static final String OSGI_EXCLUDE_PROP_FILENAME = "osgi-exclude.properties"; ////$NON-NLS-1$
 
+    private boolean ENABLE_CACHE = StringUtils.equals(System.getProperty("enable.manifest.cache", "false"), "true");
+
     private static final Collection<String> EXCLUDED_MODULES = new ArrayList<String>();
 
+    private Bundle esbBundle = Platform.getBundle(PluginChecker.ESBSE_PLUGIN_ID);
+
+    File cacheFile = null;
+
     public JobJavaScriptOSGIForESBManager(Map<ExportChoice, Object> exportChoiceMap, String contextName, String launcher,
             int statisticPort, int tracePort) {
         super(exportChoiceMap, contextName, launcher, statisticPort, tracePort);
+
+        if (esbBundle != null) {
+            IPath stateLocationPath = Platform.getStateLocation(esbBundle);
+
+            cacheFile = new File(stateLocationPath.toFile().getAbsolutePath() + File.separatorChar + "dependencyMap.cache");
+
+            FileInputStream fi = null;
+            ObjectInputStream oi = null;
+            try {
+                if (!cacheFile.exists()) {
+                    cacheFile.createNewFile();
+                } else {
+                    if (cacheFile.length() > 0) {
+                        fi = new FileInputStream(cacheFile);
+                        oi = new ObjectInputStream(fi);
+                        dependencyCacheMap = (Map) oi.readObject();
+                    }
+                }
+
+            } catch (Exception e) {
+                e.printStackTrace();
+            } finally {
+                try {
+                    if (oi != null) {
+                        oi.close();
+                    }
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
     }
 
+    Map<String, List<Object>> dependencyCacheMap = null;
     protected static final char PACKAGE_SEPARATOR = '.';
 
     private static final String JAVA = "java"; //$NON-NLS-1$
@@ -850,33 +893,37 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
     private Manifest getManifest(ExportFileResource libResource, ProcessItem processItem) throws IOException {
         Analyzer analyzer = createAnalyzer(libResource, processItem);
 
+        Set<String> imports = null;
+
         if (GlobalServiceRegister.getDefault().isServiceRegistered(IRunProcessService.class)) {
             IRunProcessService service = (IRunProcessService) GlobalServiceRegister.getDefault()
                     .getService(IRunProcessService.class);
             ITalendProcessJavaProject talendProcessJavaProject = service.getTalendJobJavaProject(processItem.getProperty());
             if (talendProcessJavaProject != null) {
                 String optional = RESOLUTION_OPTIONAL;
-                Set<String> imports = importCompiler(service, processItem);
+                imports = importCompiler(service, processItem);
                 String[] defaultPackages = analyzer.getProperty(Analyzer.IMPORT_PACKAGE).split(",");
 
                 // JDK upgrade to 11
                 imports.add("org.osgi.framework");
 
                 for (String dp : defaultPackages) {
-                    if (!imports.contains(dp) && !imports.contains(dp + optional)) {
+                    if (!imports.contains(dp) && !imports.contains(dp + RESOLUTION_OPTIONAL)) {
                         imports.add(dp);
                     }
                 }
                 imports.remove("*;resolution:=optional");
                 imports.remove("routines.system");
-                imports.remove("routines.system" + optional);
-                StringBuilder importPackage = new StringBuilder();
-                for (String packageName : imports) {
-                    importPackage.append(packageName).append(',');
+                imports.remove("routines.system" + RESOLUTION_OPTIONAL);
+
+                if (!ENABLE_CACHE) {
+                    // TESB-24730 set specific version for "javax.annotation"
+                    imports.remove("javax.annotation");
+                    imports.remove("javax.annotation" + RESOLUTION_OPTIONAL);
+                    imports.add("javax.annotation;version=\"[1.3,2)\"" + RESOLUTION_OPTIONAL);
                 }
 
-                importPackage.append("*;resolution:=optional");
-                analyzer.setProperty(Analyzer.IMPORT_PACKAGE, importPackage.toString());
+                analyzer.setProperty(Analyzer.IMPORT_PACKAGE, String.join(",", imports) + ",*;resolution:=optional");
             }
         }
 
@@ -884,7 +931,8 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         Manifest manifest = null;
         try {
             manifest = analyzer.calcManifest();
-            filterImportPackages(manifest);
+            filterPackagesCache(manifest, imports);
+
         } catch (IOException e) {
             throw e;
         } catch (Exception e) {
@@ -909,36 +957,157 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         return manifest;
     }
     
-    private void filterImportPackages(Manifest manifest) {
+    private boolean cacheManifest(URL url, String relativePath) {
+        boolean result = false;
+        ObjectOutputStream o = null;
+        Analyzer al = new Analyzer();
+        try {
+            File jarFile = new File(url.toURI());
 
-        // remove import packages which are present in private packages
+            try (InputStream is = RepositoryPlugin.getDefault().getBundle()
+                    .getEntry("/resources/osgi-ignore-calmanifest.properties").openStream();
+                    BufferedReader br = new BufferedReader(new InputStreamReader(is))) {
 
-        List<String> privatePackages = new ArrayList<String>(); 
-        String privatePackagesString = manifest.getMainAttributes().getValue(Analyzer.PRIVATE_PACKAGE);
-        if (privatePackagesString != null) {
-            String [] packages = privatePackagesString.split(",");
-            for (String p : packages) {
-                privatePackages.add(p);
+                Optional<String> rs = br.lines()
+                        .filter(fn -> StringUtils.equals(jarFile.getName(), fn) || StringUtils.equals(fn, "*")).findFirst();
+                result = rs.orElse(null) != null;
             }
-        }
-        
-        StringBuilder fileterdImportPackage = new StringBuilder();
-        String importPackagesString = manifest.getMainAttributes().getValue(Analyzer.IMPORT_PACKAGE);
-        if (importPackagesString != null) {
-            String [] packages = importPackagesString.split(",");
-            for (String p : packages) {
-                String importPackage = p.split(";")[0];
-                if (!privatePackages.contains(importPackage) || importPackage.startsWith("routines")) {
-                    fileterdImportPackage.append(p).append(",");
+
+            if (result) {
+
+                String key = jarFile.length() + jarFile.getName();
+
+                if (dependencyCacheMap == null || dependencyCacheMap.get(key) == null) {
+                    Jar bin = new Jar(jarFile);
+                    al.setJar(bin);
+                    // al.setProperty(Analyzer.IMPORT_PACKAGE, "*;resolution:=optional");
+                    // bin.putResource(relativePath, new FileResource(jarFile));
+                    Manifest manifest = al.calcManifest();
+                    String requireCapabilityString = manifest.getMainAttributes().getValue(Analyzer.REQUIRE_CAPABILITY);
+
+                    // Todo Handle requireCapabilityString to the MANIFEST.MF file ?
+//                    if (requireCapabilityString != null) {
+//
+//                    }
+                    Domain d = Domain.domain(manifest);
+                    Parameters imports = d.getImportPackage();
+                    Parameters privates = d.getPrivatePackage();
+
+                    String privatePackageString = manifest.getMainAttributes().getValue(Analyzer.PRIVATE_PACKAGE);
+                    String importPackageString = manifest.getMainAttributes().getValue(Analyzer.IMPORT_PACKAGE);
+
+                    if (StringUtils.isBlank(privatePackageString) || StringUtils.isBlank(importPackageString)) {
+                        bundleClasspathKeys.remove(key);
+                        return false;
+                    }
+
+                    List<Object> infos = new ArrayList<>();
+                    infos.add(imports.keyList());
+                    infos.add(privates.keyList());
+                    infos.add(relativePath);
+
+                    if (dependencyCacheMap == null) {
+                        dependencyCacheMap = new HashMap<>();
+                    }
+
+                    dependencyCacheMap.put(key, infos);
+
+                    if (cacheFile != null && cacheFile.exists()) {
+                        FileOutputStream f = new FileOutputStream(cacheFile);
+                        o = new ObjectOutputStream(f);
+                        o.writeObject(dependencyCacheMap);
+                    }
+                } else {
+                    if (dependencyCacheMap.get(key).get(0) == null || dependencyCacheMap.get(key).get(1) == null) {
+                        dependencyCacheMap.remove(key);
+                        bundleClasspathKeys.remove(key);
+                    }
+
                 }
             }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            try {
+                if (o != null) {
+                    o.close();
+                }
+                if (al != null) {
+                    al.close();
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
         }
-        
-        String str = fileterdImportPackage.toString();
-        if (str != null && str.length() > 0 && str.endsWith(",")) {
-            str = str.substring(0, str.length() - 1);
+
+        return result;
+
+    }
+
+    private List<String> bundleClasspathKeys = new ArrayList<>();
+
+    @SuppressWarnings("unchecked")
+    private void filterPackagesCache(Manifest manifest, Set<String> imports) {
+
+        if (!ENABLE_CACHE) {
+            bundleClasspathKeys.clear();
         }
-        manifest.getMainAttributes().putValue(Analyzer.IMPORT_PACKAGE, str);
+
+        Domain domain = Domain.domain(manifest);
+        Parameters calculatedImports = domain.getImportPackage();
+        Parameters calculatedPrivates = domain.getPrivatePackage();
+
+        Set<String> privateNonRepetitivePackages = new HashSet<>(calculatedPrivates.keySet());
+
+        Set<String> importNonRepetitivePackages = new HashSet<>(calculatedImports.keySet());
+
+        importNonRepetitivePackages.addAll(imports);
+
+        importNonRepetitivePackages = importNonRepetitivePackages.stream().map(v -> {
+            if (!StringUtils.endsWith(v, RESOLUTION_OPTIONAL)) {
+                return v + RESOLUTION_OPTIONAL;
+            }
+            return v;
+        }).collect(Collectors.toSet());
+
+        int size = bundleClasspathKeys.size();
+
+        for (int i = 0; i < size; i++) {
+            List<Object> infos = dependencyCacheMap.get(bundleClasspathKeys.get(i));
+            if (infos == null) {
+                continue;
+            }
+            List<String> parameters = (List<String>) infos.get(0);
+
+            importNonRepetitivePackages.addAll(parameters.stream().map(v -> {
+                if (!StringUtils.endsWith(v, RESOLUTION_OPTIONAL)) {
+                    return v + RESOLUTION_OPTIONAL;
+                }
+                return v;
+            }).collect(Collectors.toSet()));
+
+            privateNonRepetitivePackages.addAll((List<String>) infos.get(1));
+        }
+
+        importNonRepetitivePackages.remove("routines.system");
+        importNonRepetitivePackages.remove("routines.system" + RESOLUTION_OPTIONAL);
+
+        // TESB-24730 set specific version for "javax.annotation"
+        importNonRepetitivePackages.remove("javax.annotation");
+        importNonRepetitivePackages.remove("javax.annotation" + RESOLUTION_OPTIONAL);
+        importNonRepetitivePackages.add("javax.annotation;version=\"[1.3,2)\"" + RESOLUTION_OPTIONAL);
+
+        Set<String> fileterdImportPackage = new HashSet<>();
+
+        for (String p : importNonRepetitivePackages) {
+            if (!privateNonRepetitivePackages.contains(p.replace(RESOLUTION_OPTIONAL, "")) || p.startsWith("routines")) {
+                fileterdImportPackage.add(p);
+            }
+        }
+
+        manifest.getMainAttributes().putValue(Analyzer.PRIVATE_PACKAGE, String.join(",", privateNonRepetitivePackages));
+        manifest.getMainAttributes().putValue(Analyzer.IMPORT_PACKAGE, String.join(",", fileterdImportPackage));
     }
 
     protected Analyzer createAnalyzer(ExportFileResource libResource, ProcessItem processItem) throws IOException {
@@ -983,14 +1152,22 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
                 }
                 File dependencyFile = new File(FilesUtils.getFileRealPath(url.getPath()));
                 String relativePath = libResource.getDirectoryName() + PATH_SEPARATOR + dependencyFile.getName();
+                bundleClasspathKeys.add(dependencyFile.length() + dependencyFile.getName());
                 bundleClasspath.append(MANIFEST_ITEM_SEPARATOR).append(relativePath);
-                bin.putResource(relativePath, new FileResource(dependencyFile));
+
                 // analyzer.addClasspath(new File(url.getPath()));
                 // Add dynamic library declaration in manifest
                 if (relativePath.toLowerCase().endsWith(DLL_FILE) || relativePath.toLowerCase().endsWith(SO_FILE)) {
                     bundleNativeCode.append(libResource.getDirectoryName() + PATH_SEPARATOR + dependencyFile.getName()).append(
                             OSGI_OS_CODE);
                 }
+
+                // If don't want to enable this feature just comment out
+                if (ENABLE_CACHE && cacheManifest(url, relativePath)) {
+                    continue;
+                }
+
+                bin.putResource(relativePath, new FileResource(dependencyFile));
             }
         }
         analyzer.setProperty(Analyzer.BUNDLE_CLASSPATH, bundleClasspath.toString());
@@ -1001,11 +1178,6 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
             analyzer.setProperty(Analyzer.BUNDLE_NATIVECODE, bundleNativeCode.toString());
         }
 
-        // TESB-24730 set specific version for "javax.annotation"
-        ImportedPackageRangeReplacer r = new ImportedPackageRangeReplacer();
-        r.addRange("javax.annotation", "[1.3,2)");
-        analyzer.addBasicPlugin(r);
-
         return analyzer;
     }
 
@@ -1024,7 +1196,6 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
                 importPackages.add("org.apache.cxf.jaxrs.swagger");
             }
 
-
             if (EmfModelUtils.computeCheckElementValue("NEED_AUTH", restRequestComponent)) { //$NON-NLS-1$
                 String authType = EmfModelUtils.computeTextElementValue("AUTH_TYPE", restRequestComponent); //$NON-NLS-1$
                 if ("BASIC".equals(authType)) { //$NON-NLS-1$
@@ -1182,8 +1353,8 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
                         imports.addAll(importCompiler(service, subjobInfo.getProcessItem()));
                     }
                 }
-	        }
-    	}
+            }
+        }
         return imports;
     }
     
@@ -1219,116 +1390,4 @@ public class JobJavaScriptOSGIForESBManager extends JobJavaScriptsManager {
         }
         return imports;
     }
-
-    private class ImportedPackageRangeReplacer implements AnalyzerPlugin, Plugin {
-
-        private Set<Range> ranges = new TreeSet<>();
-
-        public void addRange(String packageName, String packageVersion) {
-            ranges.add(new Range(packageName, packageVersion));
-        }
-
-        /**
-         * Analyzes the jar and update the version range.
-         *
-         * @param analyzer the analyzer
-         * @return {@code false}
-         * @throws Exception if the analaysis fails.
-         */
-        @Override
-        public boolean analyzeJar(Analyzer analyzer) throws Exception {
-
-            if (analyzer.getReferred() == null) {
-                return false;
-            }
-
-            for (Map.Entry<Descriptors.PackageRef, Attrs> entry : analyzer.getReferred().entrySet()) {
-                for (Range range : ranges) {
-                    if (range.matches(entry.getKey().getFQN())) {
-                        String value = range.getRange(analyzer);
-                        if (value != null) {
-                            entry.getValue().put("version", value);
-                        }
-                    }
-                }
-            }
-            return false;
-        }
-
-
-        private class Range implements Comparable<Range> {
-            final String name;
-            final String value;
-            final Pattern regex;
-
-            private String foundRange;
-
-            private Range(String name, String value) {
-                this.name = name;
-                this.value = value;
-                this.regex = Pattern.compile(name.trim().replace(".", "\\.").replace("*", ".*"));
-            }
-
-            private boolean matches(String pck) {
-                return regex.matcher(pck).matches();
-            }
-
-            private String getRange(Analyzer analyzer) throws Exception {
-                if (foundRange != null) {
-                    return foundRange;
-                }
-                if (null == value || value.isEmpty()) {
-                    for (Jar jar : analyzer.getClasspath()) {
-                        if (isProvidedByJar(jar) && jar.getVersion() != null) {
-                            foundRange = jar.getVersion();
-                            return jar.getVersion();
-                        }
-                    }
-                    return null;
-                } else {
-                    return value;
-                }
-            }
-
-            private boolean isProvidedByJar(Jar jar) {
-                for (String s : jar.getPackages()) {
-                    if (matches(s)) {
-                        return true;
-                    }
-                }
-                return false;
-            }
-
-            @Override
-            public int compareTo(Range o) {
-                return Integer.compare(this.regex.pattern().length(), o.regex.pattern().length());
-            }
-
-            @Override
-            public boolean equals(Object o) {
-                if (this == o) {
-                    return true;
-                }
-                if (o == null || getClass() != o.getClass()) {
-                    return false;
-                }
-                Range range = (Range) o;
-                return Objects.equals(name, range.name) &&
-                        Objects.equals(value, range.value);
-            }
-
-            @Override
-            public int hashCode() {
-                return Objects.hashCode(name + value);
-            }
-        }
-
-        @Override
-        public void setReporter(Reporter processor) {
-        }
-
-        @Override
-        public void setProperties(Map<String, String> map) throws Exception {
-        }
-    }
-}
+}