fix(TUP-30849):Improve build Job performance (#4315)

* Jding/tup 30849 improve build job performance (#4197) (#4198)

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849



* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849


* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

* fix(TUP-30849):Improve build Job performance
https://jira.talendforge.org/browse/TUP-30849

Signed-off-by: jding-tlnd <jding@talend.com>

Conflicts:
	main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java
	test/plugins/org.talend.core.test/src/org/talend/designer/runprocess/ProcessorUtilitiesTest.java

Conflicts:
	test/plugins/org.talend.core.test/src/org/talend/designer/runprocess/ProcessorUtilitiesTest.java

* fix(TUP-30849):Improve build Job performance (#4310)

https://jira.talendforge.org/browse/TUP-30849

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/IProcess.java

@@ -12,6 +12,7 @@
 // ============================================================================
 package org.talend.core.model.process;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -133,4 +134,8 @@ public interface IProcess extends IElement {
     public String getComponentsType();
 
     public INode getNodeByUniqueName(String uniqueName);
+
+    default List<? extends INode> getProcessNodes() {
+        return new ArrayList<INode>();
+    }
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/process/JobInfo.java

@@ -67,6 +67,8 @@ public class JobInfo {
 
     private boolean needUnloadProcessor;
 
+    private int includeESBFlag = 0;
+
     public JobInfo(String jobId, String contextName, String version) {
         this.jobId = jobId;
         this.contextName = contextName;
@@ -572,4 +574,13 @@ public class JobInfo {
     public boolean isNeedUnloadProcessor() {
         return this.needUnloadProcessor;
     }
+
+    public int getIncludeESBFlag() {
+        return includeESBFlag;
+    }
+
+    public void setIncludeESBFlag(int includeESBFlag) {
+        this.includeESBFlag = includeESBFlag;
+    }
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/utils/ContextParameterUtils.java

@@ -36,6 +36,9 @@ import org.apache.oro.text.regex.Perl5Compiler;
 import org.apache.oro.text.regex.Perl5Matcher;
 import org.apache.oro.text.regex.Perl5Substitution;
 import org.apache.oro.text.regex.Util;
+import org.eclipse.core.runtime.Platform;
+import org.eclipse.core.runtime.preferences.IEclipsePreferences;
+import org.eclipse.core.runtime.preferences.InstanceScope;
 import org.talend.commons.utils.PasswordEncryptUtil;
 import org.talend.core.GlobalServiceRegister;
 import org.talend.core.language.ECodeLanguage;
@@ -48,6 +51,8 @@ import org.talend.core.model.metadata.types.JavaTypesManager;
 import org.talend.core.model.process.IContext;
 import org.talend.core.model.process.IContextManager;
 import org.talend.core.model.process.IContextParameter;
+import org.talend.core.model.repository.IRepositoryPrefConstants;
+import org.talend.core.prefs.ITalendCorePrefConstants;
 import org.talend.core.runtime.CoreRuntimePlugin;
 import org.talend.core.runtime.services.IGenericDBService;
 import org.talend.core.utils.TalendQuoteUtils;
@@ -614,7 +619,12 @@ public final class ContextParameterUtils {
     }
 
     private static boolean isAllowSpecificCharacters() {
-        return CoreRuntimePlugin.getInstance().getProjectPreferenceManager().isAllowSpecificCharacters();
+        if (Platform.isRunning()) {
+            return CoreRuntimePlugin.getInstance().getProjectPreferenceManager().isAllowSpecificCharacters();
+        } else {
+            IEclipsePreferences coreUIPluginNode = new InstanceScope().getNode(ITalendCorePrefConstants.CoreUIPlugin_ID);
+            return coreUIPluginNode.getBoolean(IRepositoryPrefConstants.ALLOW_SPECIFIC_CHARACTERS_FOR_SCHEMA_COLUMNS, false);
+        }
     }
 
     public static boolean isEmptyParameter(String source) {

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/runtime/process/TalendProcessOptionConstants.java

@@ -80,4 +80,13 @@ public interface TalendProcessOptionConstants {
      */
     public static final int MODULES_WITH_CODESJAR = 1 << 6;
 
+    /**
+     * flag for check if is ESB job
+     */
+    public static final int ISESB_CHECKED = 1;
+
+    public static final int ISESB_CHILDREN_INCLUDE = 1 << 1;
+
+    public static final int ISESB_CURRENT_INCLUDE = 1 << 2;
+
 }

main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/ui/IJobletProviderService.java

@@ -147,4 +147,6 @@ public interface IJobletProviderService extends IService {
 
     public void updateJobleModifiedRelated(Item item, String oldName, String newName);
 
+    public IProcess getJobletGEFProcessFromNode(INode node);
+
 }

main/plugins/org.talend.core/src/main/java/org/talend/designer/runprocess/ProcessorUtilities.java

@@ -25,8 +25,10 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
@@ -98,10 +100,8 @@ import org.talend.core.model.process.JobInfo;
 import org.talend.core.model.process.ProcessUtils;
 import org.talend.core.model.process.ReplaceNodesInProcessProvider;
 import org.talend.core.model.properties.Item;
-import org.talend.core.model.properties.JobletProcessItem;
 import org.talend.core.model.properties.ProcessItem;
 import org.talend.core.model.properties.Property;
-import org.talend.core.model.relationship.Relation;
 import org.talend.core.model.relationship.RelationshipItemBuilder;
 import org.talend.core.model.repository.ERepositoryObjectType;
 import org.talend.core.model.repository.IRepositoryObject;
@@ -192,7 +192,7 @@ public class ProcessorUtilities {
 
     private static final Set<ModuleNeeded> retrievedJarsForCurrentBuild = new HashSet<ModuleNeeded>();
 
-    private static final Set<String> esbJobs = new HashSet<String>();
+    private static final Map<String, Integer> esbJobs = new HashMap<String, Integer>();
 
     private static boolean isDebug = false;
 
@@ -439,6 +439,81 @@ public class ProcessorUtilities {
         return false;
     }
 
+    public static boolean checkProcessLoopDependencies(IProcess mainProcess, String id, String version,
+            LinkedList<String> pathlink, Map<String, String> idToLatestVersion) {
+        if (ItemCacheManager.LATEST_VERSION.contains(version)) {
+            if (idToLatestVersion.get(id) == null) {
+                ProcessItem processItem = ItemCacheManager.getProcessItem(id);
+                version = processItem.getProperty().getVersion();
+                idToLatestVersion.put(id, version);
+            } else {
+                version = idToLatestVersion.get(id);
+            }
+        }
+        String pathNode = id + "-" + version;
+        if (pathlink.contains(pathNode)) {
+            return true;
+        }
+        pathlink.add(pathNode);
+
+        boolean hasLoop = false;
+        List<? extends INode> processNodes = mainProcess.getProcessNodes();
+        for (INode node : processNodes) {
+            if (!node.isActivate()) {
+                continue;
+            }
+
+            IElementParameter processIdParam = node.getElementParameter("PROCESS_TYPE_PROCESS");
+            if (processIdParam != null && StringUtils.isNotBlank((String) processIdParam.getValue())) {
+                String jobIds = (String) processIdParam.getValue();
+                String subNodeversion = (String) node.getElementParameter("PROCESS_TYPE_VERSION").getValue();
+                for (String jobId : jobIds.split(ProcessorUtilities.COMMA)) {
+                    if (StringUtils.isBlank(jobId)) {
+                        continue;
+                    }
+                    ProcessItem processItem = ItemCacheManager.getProcessItem(jobId, subNodeversion);
+                    if (processItem != null) {
+                        IDesignerCoreService service = CorePlugin.getDefault().getDesignerCoreService();
+                        IProcess subProcess = service.getProcessFromProcessItem(processItem);
+                        if (subProcess != null) {
+                            hasLoop = checkProcessLoopDependencies(subProcess, jobId, subNodeversion, pathlink,
+                                    idToLatestVersion);
+                        }
+                    }
+                    if (hasLoop) {
+                        break;
+                    }
+                }
+            } else {
+                if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
+                    IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
+                            .getService(IJobletProviderService.class);
+                    if (jobletService != null) {
+                        IProcess jobletProcess = jobletService.getJobletGEFProcessFromNode(node);
+                        if (jobletProcess != null) {
+                            String jobletId = jobletProcess.getId();
+                            IElementParameter projectTecNameParam = jobletProcess.getElementParameter("PROJECT_TECHNICAL_NAME");
+                            if (projectTecNameParam != null && StringUtils.isNotBlank((String) projectTecNameParam.getValue())) {
+                                jobletId = projectTecNameParam.getValue() + ":" + jobletId;
+                            }
+                            hasLoop = checkProcessLoopDependencies(jobletProcess, jobletId, jobletProcess.getVersion(), pathlink,
+                                    idToLatestVersion);
+                        }
+                    }
+                }
+            }
+            if (hasLoop) {
+                break;
+            }
+        }
+
+        if (!hasLoop) {
+            pathlink.removeLast();
+        }
+
+        return hasLoop;
+    }
+
     private static IProcessor generateCode(IProcessor processor2, JobInfo jobInfo, String selectedContextName,
             boolean statistics, boolean trace, boolean needContext, int option, IProgressMonitor progressMonitor)
             throws ProcessorException {
@@ -538,11 +613,10 @@ public class ProcessorUtilities {
         jobInfo.setProcessor(processor);
 
         if (isMainJob && selectedProcessItem != null) {
-            Relation mainRelation = new Relation();
-            mainRelation.setId(jobInfo.getJobId());
-            mainRelation.setVersion(jobInfo.getJobVersion());
-            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+            Property property = selectedProcessItem.getProperty();
+            String jobId = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel() + ":" + property.getId();
+            hasLoopDependency = checkProcessLoopDependencies(currentProcess, jobId, property.getVersion(),
+                    new LinkedList<String>(), new HashMap<String, String>());
             // clean the previous code in case it has deleted subjob
             cleanSourceFolder(progressMonitor, currentProcess, processor);
         }
@@ -645,233 +719,6 @@ public class ProcessorUtilities {
         return processor;
     }
 
-    public static boolean checkLoopDependencies(Relation mainJobInfo, Map<String, String> idToLastestVersionMap)
-            throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = getItemsRelation(mainJobInfo, idToLastestVersionMap);
-        List<Relation> relationChecked = new ArrayList<>();
-        relationChecked.add(mainJobInfo);
-        return checkLoopDependencies(mainJobInfo, mainJobInfo, itemsJobRelatedTo, relationChecked, idToLastestVersionMap);
-    }
-
-    private static boolean checkLoopDependencies(Relation mainRelation, Relation currentRelation,
-            List<Relation> itemsJobRelatedTo,
-            List<Relation> relationChecked, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        boolean hasDependency = false;
-        for (Relation relation : itemsJobRelatedTo) {
-            try {
-                // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                Map<String, Set<String>> actTrunjobHM = getActivateTRunjobMap(currentRelation.getId(),
-                        currentRelation.getVersion());
-                if (actTrunjobHM.get(relation.getId()) == null
-                        || !actTrunjobHM.get(relation.getId()).contains(relation.getVersion())) {
-                    continue;
-                }
-            } catch (Exception e) {
-                throw new ProcessorException(e);
-            }
-
-            hasDependency = relation.getId().equals(mainRelation.getId())
-                    && relation.getVersion().equals(mainRelation.getVersion());
-            if (!hasDependency) {
-                List<Relation> itemsChildJob = getItemsRelation(relation, idToLastestVersionMap);
-                if (!relationChecked.contains(relation)) {
-                    relationChecked.add(relation);
-                    hasDependency = checkLoopDependencies(mainRelation, relation, itemsChildJob, relationChecked,
-                            idToLastestVersionMap);
-                }
-                if (!hasDependency) {
-                    for (Relation childRelation : itemsChildJob) {
-
-                        try {
-                            // means the tRunjob deactivate, or one of the specific version tRunjon deactivate, skip
-                            Map<String, Set<String>> activateTRunjobMap = getActivateTRunjobMap(relation.getId(),
-                                    relation.getVersion());
-                            if (activateTRunjobMap.get(childRelation.getId()) == null
-                                    || !activateTRunjobMap.get(childRelation.getId()).contains(childRelation.getVersion())) {
-                                continue;
-                            }
-                        } catch (Exception e) {
-                            throw new ProcessorException(e);
-                        }
-
-                        hasDependency = checkLoopDependencies(childRelation, idToLastestVersionMap);
-                        if (hasDependency) {
-                            break;
-                        }
-                    }
-                }
-            }
-            if (hasDependency) {
-                break;
-            }
-        }
-
-        return hasDependency;
-    }
-
-    private static Map<String, Set<String>> getActivateTRunjobMap(String id, String version) throws PersistenceException {
-        Map<String, Set<String>> actTrunjobHM = new HashMap<String, Set<String>>();
-        ProcessType processType = null;
-        try {
-            IRepositoryViewObject currentJobObject = ProxyRepositoryFactory.getInstance().getSpecificVersion(id, version, true);
-            if (currentJobObject != null) {
-                Item item = currentJobObject.getProperty().getItem();
-                if (item instanceof ProcessItem) {
-                    processType = ((ProcessItem) item).getProcess();
-                } else if (item instanceof JobletProcessItem) {
-                    processType = ((JobletProcessItem) item).getJobletProcess();
-                }
-            }
-        } catch (PersistenceException e) {
-            ExceptionHandler.process(e);
-        }
-        if (processType != null) {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-
-            List<String> jobletsComponentsList = new ArrayList<String>();
-            IComponentsFactory componentsFactory = null;
-            if (GlobalServiceRegister.getDefault().isServiceRegistered(IComponentsService.class)) {
-                IComponentsService compService = GlobalServiceRegister.getDefault()
-                        .getService(IComponentsService.class);
-                if (compService != null) {
-                    componentsFactory = compService.getComponentsFactory();
-                    for (IComponent component : componentsFactory.readComponents()) {
-                        if (component.getComponentType() == EComponentType.JOBLET) {
-                            jobletsComponentsList.add(component.getName());
-                        }
-                    }
-                }
-            }
-
-            String jobletPaletteType = null;
-            String frameWork = processType.getFramework();
-            if (StringUtils.isBlank(frameWork)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_DI.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARK.getName();
-            } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
-                jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
-            }
-
-            for (Object nodeObject : processType.getNode()) {
-                NodeType node = (NodeType) nodeObject;
-                // not tRunjob && not joblet then continue
-                if (!node.getComponentName().equals("tRunJob") && !jobletsComponentsList.contains(node.getComponentName())) { // $NON-NLS-1$
-                    continue;
-                }
-                boolean nodeActivate = true;
-                String processIds = null;
-                String processVersion = null;
-                for (Object elementParam : node.getElementParameter()) {
-                    ElementParameterType elemParamType = (ElementParameterType) elementParam;
-                    if ("PROCESS:PROCESS_TYPE_PROCESS".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processIds = elemParamType.getValue();
-                        if (StringUtils.isNotBlank(processIds)) {
-                            for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                                if (actTrunjobHM.get(jobId) == null) {
-                                    actTrunjobHM.put(jobId, new HashSet<String>());
-                                }
-                            }
-                        }
-                    } else if ("PROCESS:PROCESS_TYPE_VERSION".equals(elemParamType.getName()) // $NON-NLS-1$
-                            || "PROCESS_TYPE_VERSION".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        processVersion = elemParamType.getValue();
-                    } else if ("ACTIVATE".equals(elemParamType.getName())) { // $NON-NLS-1$
-                        nodeActivate = Boolean.parseBoolean(elemParamType.getValue());
-                    }
-                }
-
-                if (nodeActivate) {
-                    if (StringUtils.isNotBlank(processIds)) {
-                        for (String jobId : processIds.split(ProcessorUtilities.COMMA)) {
-                            String actualVersion = processVersion;
-                            if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                for (Project project : allProjects) {
-                                    IRepositoryViewObject lastVersion = null;
-                                    lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobId);
-                                    if (lastVersion != null) {
-                                        actualVersion = lastVersion.getVersion();
-                                        break;
-                                    }
-                                }
-                            }
-                            if (actTrunjobHM.get(jobId) != null) {
-                                actTrunjobHM.get(jobId).add(actualVersion);
-                            }
-
-                        }
-                    } else if (componentsFactory != null && jobletPaletteType != null) {
-                        // for joblet
-                        IComponent cc = componentsFactory.get(node.getComponentName(), jobletPaletteType);
-                        if (GlobalServiceRegister.getDefault().isServiceRegistered(IJobletProviderService.class)) {
-                            IJobletProviderService jobletService = GlobalServiceRegister.getDefault()
-                                    .getService(IJobletProviderService.class);
-                            Property property = jobletService.getJobletComponentItem(cc);
-                            if (property != null && StringUtils.isNotBlank(property.getId())) {
-                                String jobletId = property.getId();
-                                if (actTrunjobHM.get(jobletId) == null) {
-                                    actTrunjobHM.put(jobletId, new HashSet<String>());
-                                }
-                                String actualVersion = processVersion;
-                                if (RelationshipItemBuilder.LATEST_VERSION.equals(processVersion)) {
-                                    for (Project project : allProjects) {
-                                        IRepositoryViewObject lastVersion = null;
-                                        lastVersion = ProxyRepositoryFactory.getInstance().getLastVersion(project, jobletId);
-                                        if (lastVersion != null) {
-                                            actualVersion = lastVersion.getVersion();
-                                            break;
-                                        }
-                                    }
-                                }
-
-                                actTrunjobHM.get(jobletId).add(actualVersion);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        return actTrunjobHM;
-    }
-
-    private static List<Relation> getItemsRelation(Relation mainJobInfo, Map<String, String> idToLastestVersionMap) throws ProcessorException {
-        List<Relation> itemsJobRelatedTo = new ArrayList<Relation>();
-        try {
-            List<Project> allProjects = new ArrayList<Project>();
-            allProjects.add(ProjectManager.getInstance().getCurrentProject());
-            allProjects.addAll(ProjectManager.getInstance().getAllReferencedProjects());
-            RelationshipItemBuilder instance = RelationshipItemBuilder.getInstance();
-            if (instance != null) {
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOB_RELATION));
-                itemsJobRelatedTo.addAll(instance.getItemsChildRelatedTo(mainJobInfo.getId(), mainJobInfo.getVersion(),
-                        mainJobInfo.getType(), RelationshipItemBuilder.JOBLET_RELATION));
-                for (Relation relation : itemsJobRelatedTo) {
-                    if (relation.getVersion().equals(RelationshipItemBuilder.LATEST_VERSION)) {
-                        if (idToLastestVersionMap.containsKey(relation.getId())) {
-                            relation.setVersion(idToLastestVersionMap.get(relation.getId()));
-                        } else {
-                            for (Project project : allProjects) {
-                                IRepositoryViewObject lastVersion =
-                                        ProxyRepositoryFactory.getInstance().getLastVersion(project, relation.getId());
-                                if (lastVersion != null) {
-                                    relation.setVersion(lastVersion.getVersion());
-                                    idToLastestVersionMap.put(relation.getId(), relation.getVersion());
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (PersistenceException e) {
-            throw new ProcessorException(e);
-        }
-
-        return itemsJobRelatedTo;
-    }
 
     private static void setNeededResources(final Map<String, Object> argumentsMap, JobInfo jobInfo) {
         argumentsMap.put(TalendProcessArgumentConstant.ARG_NEED_XMLMAPPINGS,
@@ -1265,11 +1112,9 @@ public class ProcessorUtilities {
             }
 
             if (isMainJob && selectedProcessItem != null) {
-                Relation mainRelation = new Relation();
-                mainRelation.setId(jobInfo.getJobId());
-                mainRelation.setVersion(jobInfo.getJobVersion());
-                mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-                hasLoopDependency = checkLoopDependencies(mainRelation, new HashMap<String, String>());
+                Property property = selectedProcessItem.getProperty();
+                hasLoopDependency = checkProcessLoopDependencies(currentProcess, property.getId(), property.getVersion(),
+                        new LinkedList<String>(), new HashMap<String, String>());
                 // clean the previous code in case it has deleted subjob
                 cleanSourceFolder(progressMonitor, currentProcess, processor);
             }
@@ -1800,13 +1645,16 @@ public class ProcessorUtilities {
                         }
                     }
                 }
-                if (isEsbComponentName(componentName)) {
-                    addEsbJob(jobInfo);
-                }
             }
         }
     }
 
+    private static void recordESBIncludingFlag(JobInfo jobInfo, int esbIncludingOption) {
+        int includeESBFlag = jobInfo.getIncludeESBFlag();
+        includeESBFlag |= esbIncludingOption;
+        jobInfo.setIncludeESBFlag(includeESBFlag);
+    }
+
     static void setGenerationInfoWithChildrenJob(INode node, JobInfo jobInfo, final JobInfo subJobInfo) {
         final LastGenerationInfo generationInfo = LastGenerationInfo.getInstance();
 
@@ -1915,6 +1763,13 @@ public class ProcessorUtilities {
 
     static boolean hasLoopDependency = false;
 
+    private static void resetBuildFlagsAndCaches() {
+        jobList.clear();
+        esbJobs.clear();
+        hasLoopDependency = false;
+        mainJobInfo = null;
+    }
+
     /**
      * This function will generate the code of the process and all of this sub process.
      *
@@ -1929,15 +1784,10 @@ public class ProcessorUtilities {
         if (monitors == null) {
             monitor = new NullProgressMonitor();
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         JobInfo jobInfo = new JobInfo(processName, contextName, version);
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1957,14 +1807,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(processId, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor process = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return process;
     }
 
@@ -1979,14 +1824,9 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2001,15 +1841,10 @@ public class ProcessorUtilities {
         }
         JobInfo jobInfo = new JobInfo(process, contextName, version);
         jobInfo.setApplyContextToChildren(applyContextToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result =
                 generateCode(jobInfo, contextName, statistics, trace, needContext, GENERATE_ALL_CHILDS, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2033,14 +1868,9 @@ public class ProcessorUtilities {
                 ProcessUtils.isOptionChecked(argumentsMap, TalendProcessArgumentConstant.ARG_NEED_CONTEXT);
         int option = ProcessUtils.getOptionValue(argumentsMap, TalendProcessArgumentConstant.ARG_GENERATE_OPTION, 0);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor result = generateCode(jobInfo, contextName, statistics, trace, needContext, option, monitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return result;
     }
 
@@ -2058,28 +1888,18 @@ public class ProcessorUtilities {
             JobInfo jobInfo = new JobInfo(process, contextName, version);
             jobInfo.setContext(context);
             jobInfo.setApplyContextToChildren(applyContextToChildren);
-            jobList.clear();
-            esbJobs.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
             result = generateCode(jobInfo, contextName, statistics, trace, true, GENERATE_ALL_CHILDS, monitor);
-            jobList.clear();
-            hasLoopDependency = false;
-            mainJobInfo = null;
+            resetBuildFlagsAndCaches();
         }
         return result;
     }
 
     public static IProcessor generateCode(ProcessItem process, String contextName, boolean statistics, boolean trace)
             throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue = generateCode(process, contextName, statistics, trace, false);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2107,29 +1927,19 @@ public class ProcessorUtilities {
             jobInfo = new JobInfo(process, context);
         }
         jobInfo.setApplyContextToChildren(applyToChildren);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, contextProperties,
                 GENERATE_ALL_CHILDS, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
     public static IProcessor generateCode(IProcess process, IContext context, boolean statistics, boolean trace,
             boolean properties) throws ProcessorException {
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor returnValue =
                 generateCode(process, context, statistics, trace, properties, new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return returnValue;
     }
 
@@ -2157,15 +1967,10 @@ public class ProcessorUtilities {
         } else {
             jobInfo = new JobInfo(process, context);
         }
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
@@ -2208,15 +2013,10 @@ public class ProcessorUtilities {
                 + (jobInfo.getJobName() != null ? jobInfo.getJobName() : jobInfo.getJobId());
         TimeMeasure.begin(timeMeasureGenerateCodesId);
 
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(processor, jobInfo, context.getName(), statistics, trace, properties,
                 GENERATE_ALL_CHILDS, progressMonitor);
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
 
         TimeMeasure.end(timeMeasureGenerateCodesId);
         // if active before, not disable and active still.
@@ -2239,15 +2039,10 @@ public class ProcessorUtilities {
         }
         // achen modify to fix 0006107
         JobInfo jobInfo = new JobInfo(process, context);
-        jobList.clear();
-        esbJobs.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         IProcessor genCode = generateCode(jobInfo, context.getName(), statistics, trace, properties, option,
                 new NullProgressMonitor());
-        jobList.clear();
-        hasLoopDependency = false;
-        mainJobInfo = null;
+        resetBuildFlagsAndCaches();
         return genCode;
     }
 
@@ -2611,6 +2406,8 @@ public class ProcessorUtilities {
         } else if (frameWork.equals(HadoopConstants.FRAMEWORK_SPARK_STREAMING)) {
             jobletPaletteType = ComponentCategory.CATEGORY_4_SPARKSTREAMING.getName();
         }
+
+        boolean hasChildrenIncludeESB = false;
         for (NodeType node : nodes) {
             boolean activate = true;
             // check if node is active at least.
@@ -2625,8 +2422,8 @@ public class ProcessorUtilities {
                 continue;
             }
 
-            if (isEsbComponentName(node.getComponentName())) {
-                addEsbJob(parentJobInfo);
+            if (!firstChildOnly && isEsbComponentName(node.getComponentName())) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE);
             }
 
             boolean isCTalendJob = "cTalendJob".equalsIgnoreCase(node.getComponentName());
@@ -2655,6 +2452,18 @@ public class ProcessorUtilities {
                                 jobInfo.setFatherJobInfo(parentJobInfo);
                                 if (!firstChildOnly) {
                                     getAllJobInfo(processItem.getProcess(), jobInfo, jobInfos, firstChildOnly, includeJoblet);
+                                    if (jobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
+                                }
+                            } else {
+                                Optional<JobInfo> infoOptional = jobInfos.stream().filter(info -> info.equals(jobInfo))
+                                        .findFirst();
+                                if (infoOptional.isPresent()) {
+                                    JobInfo matchJobInfo = infoOptional.get();
+                                    if (matchJobInfo.getIncludeESBFlag() >= 2) {
+                                        hasChildrenIncludeESB = true;
+                                    }
                                 }
                             }
                         }
@@ -2706,6 +2515,16 @@ public class ProcessorUtilities {
                 }
             }
         }
+
+        // checked done set the esb including option
+        if (!firstChildOnly) {
+            recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHECKED);
+            if (hasChildrenIncludeESB) {
+                recordESBIncludingFlag(parentJobInfo, TalendProcessOptionConstants.ISESB_CHILDREN_INCLUDE);
+            }
+            esbJobs.put(esbJobKey(parentJobInfo.getJobId(), parentJobInfo.getJobVersion()), parentJobInfo.getIncludeESBFlag());
+        }
+
         return jobInfos;
     }
 
@@ -2922,9 +2741,21 @@ public class ProcessorUtilities {
     }
 
     public static boolean isEsbJob(IProcess process, boolean checkCurrentProcess) {
+        // get includeESBFlag from cache
+        if (process instanceof IProcess2) {
+            Property property = ((IProcess2) process).getProperty();
+            String esbJobKey = esbJobKey(property.getId(), property.getVersion());
+            if (esbJobs.get(esbJobKey) != null) {
+                Integer esbOptions = esbJobs.get(esbJobKey);
+                if (BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CHECKED)) {
+                    return checkCurrentProcess
+                            ? BitwiseOptionUtils.containOption(esbOptions, TalendProcessOptionConstants.ISESB_CURRENT_INCLUDE)
+                            : esbOptions > 2;
+                }
+            }
+        }
 
         if (process instanceof IProcess2) {
-
             if (checkCurrentProcess) {
                 for (INode n : process.getGraphicalNodes()) {
                     if (isEsbComponentName(n.getComponent().getName())) {
@@ -2950,17 +2781,6 @@ public class ProcessorUtilities {
         return false;
     }
 
-    private static void addEsbJob(JobInfo jobInfo) {
-        if (esbJobs.contains(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()))) {
-            return;
-        }
-        
-        esbJobs.add(esbJobKey(jobInfo.getJobId(), jobInfo.getJobVersion()));
-        if (jobInfo.getFatherJobInfo() != null) {
-            addEsbJob(jobInfo.getFatherJobInfo());
-        }
-    }
-
     private static String esbJobKey(String processId, String version) {
         return processId + "_" + version;
     }

test/plugins/org.talend.core.test/src/org/talend/designer/runprocess/ProcessorUtilitiesTest.java

@@ -15,7 +15,6 @@ package org.talend.designer.runprocess;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.anyObject;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -30,7 +29,6 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -40,11 +38,9 @@ import java.util.jar.JarInputStream;
 import java.util.stream.Collectors;
 
 import org.apache.logging.log4j.Level;
-import org.eclipse.core.runtime.Path;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.talend.commons.exception.PersistenceException;
 import org.talend.core.model.components.EComponentType;
 import org.talend.core.model.components.IComponent;
 import org.talend.core.model.general.ModuleNeeded;
@@ -56,21 +52,8 @@ import org.talend.core.model.process.IElementParameter;
 import org.talend.core.model.process.INode;
 import org.talend.core.model.process.IProcess;
 import org.talend.core.model.process.JobInfo;
-import org.talend.core.model.process.ProcessUtils;
-import org.talend.core.model.properties.ProcessItem;
-import org.talend.core.model.properties.PropertiesFactory;
-import org.talend.core.model.properties.Property;
-import org.talend.core.model.relationship.Relation;
-import org.talend.core.model.relationship.RelationshipItemBuilder;
-import org.talend.core.model.repository.IRepositoryViewObject;
 import org.talend.core.model.runprocess.shadow.ObjectElementParameter;
-import org.talend.core.repository.model.ProxyRepositoryFactory;
 import org.talend.core.runtime.process.LastGenerationInfo;
-import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
-import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
-import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
-import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
-import org.talend.repository.ProjectManager;
 
 /**
  * DOC ggu class global comment. Detailled comment
@@ -448,102 +431,6 @@ public class ProcessorUtilitiesTest {
         assertTrue("DB node in job. shoud be metadata dynamic", ProcessorUtilities.hasMetadataDynamic(proc, null));
     }
 
-    @Test
-    public void testCheckLoopDependencies() {
-        String projectTecLabel = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
-        ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
-        RelationshipItemBuilder relationshipItemBuilder = RelationshipItemBuilder.getInstance();
-
-        ProcessItem item = prepareProcessItem(factory.getNextId(), "test", "0.1");
-        ProcessItem item1 = prepareProcessItem(factory.getNextId(), "test1", "0.1");
-        ProcessItem item2 = prepareProcessItem(factory.getNextId(), "test2", "0.1");
-        prepareTRunjobNode(item, projectTecLabel, item1.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-        prepareTRunjobNode(item1, projectTecLabel, item2.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-        prepareTRunjobNode(item2, projectTecLabel, item.getProperty().getId(), RelationshipItemBuilder.LATEST_VERSION);
-
-        IRepositoryViewObject repositoryObject = null;
-        IRepositoryViewObject repositoryObject1 = null;
-        IRepositoryViewObject repositoryObject2 = null;
-        try {
-            factory.create(item, new Path(""));
-            factory.create(item1, new Path(""));
-            factory.create(item2, new Path(""));
-
-            repositoryObject = factory.getSpecificVersion(item.getProperty().getId(), item.getProperty().getVersion(), true);
-            repositoryObject1 = factory.getSpecificVersion(item1.getProperty().getId(), item1.getProperty().getVersion(), true);
-            repositoryObject2 = factory.getSpecificVersion(item2.getProperty().getId(), item2.getProperty().getVersion(), true);
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject.getProperty().getItem());
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject1.getProperty().getItem());
-            relationshipItemBuilder.addOrUpdateItem(repositoryObject2.getProperty().getItem());
-
-            Relation mainRelation = new Relation();
-            mainRelation.setId(repositoryObject.getProperty().getId());
-            mainRelation.setVersion(repositoryObject.getProperty().getVersion());
-            mainRelation.setType(RelationshipItemBuilder.JOB_RELATION);
-            // job-->job1-->job2-->job hasLoop==true
-            boolean hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
-            assertTrue(hasLoop);
-
-            // job-->job1-->job2-->job(tRunjob deactivate)
-            ProcessItem jobItem2 = (ProcessItem) repositoryObject2.getProperty().getItem();
-            for (Object nodeObject : jobItem2.getProcess().getNode()) {
-                NodeType node = (NodeType) nodeObject;
-                if (!node.getComponentName().equals("tRunJob")) { // $NON-NLS-1$
-                    continue;
-                }
-                ElementParameterType actParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-                actParam.setField("CHECK");
-                actParam.setName("ACTIVATE");
-                actParam.setValue("false");
-                node.getElementParameter().add(actParam);
-            }
-            factory.save(jobItem2, false);
-            hasLoop = ProcessorUtilities.checkLoopDependencies(mainRelation, new HashMap<String, String>());
-            assertFalse(hasLoop);
-
-        } catch (Exception e) {
-            e.printStackTrace();
-            fail("Test CheckLoopDependencies failure.");
-        } finally {
-            try {
-                factory.deleteObjectPhysical(repositoryObject);
-                factory.deleteObjectPhysical(repositoryObject1);
-                factory.deleteObjectPhysical(repositoryObject2);
-            } catch (PersistenceException e) {
-                e.printStackTrace();
-                fail("Test CheckLoopDependencies failure.");
-            }
-        }
-    }
-
-    private ProcessItem prepareProcessItem(String id, String label, String version) {
-        Property property = PropertiesFactory.eINSTANCE.createProperty();
-        ProcessItem item = PropertiesFactory.eINSTANCE.createProcessItem();
-        ProcessType process = TalendFileFactory.eINSTANCE.createProcessType();
-        item.setProperty(property);
-        item.setProcess(process);
-        property.setId(id);
-        property.setLabel(label);
-        property.setVersion(version);
-        return item;
-    }
-
-    private void prepareTRunjobNode(ProcessItem item, String projectLabel, String subjobId, String subjobVersion) {
-        NodeType node = TalendFileFactory.eINSTANCE.createNodeType();
-        node.setComponentName("tRunJob");
-        item.getProcess().getNode().add(node);
-        ElementParameterType versionParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-        versionParam.setField("TECHNICAL");
-        versionParam.setName("PROCESS:PROCESS_TYPE_VERSION");
-        versionParam.setValue(subjobVersion);
-        node.getElementParameter().add(versionParam);
-        ElementParameterType jobIdParam = TalendFileFactory.eINSTANCE.createElementParameterType();
-        jobIdParam.setField("TECHNICAL");
-        jobIdParam.setName("PROCESS:PROCESS_TYPE_PROCESS");
-        jobIdParam.setValue(projectLabel + ProcessUtils.PROJECT_ID_SEPARATOR + subjobId);
-        node.getElementParameter().add(jobIdParam);
-    }
-    
     @Ignore
     @Test
     public void testWriteLog4j2ConfToFile() throws IOException {