jprdonnelly/tcommon-studio-se
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(TUP-27590):Relationship.index might get really big (#3424) (#3501)

Browse Source 
* fix(TUP-27590):Relationship.index might get really big
https://jira.talendforge.org/browse/TUP-27590

* fix(TUP-27590):Relationship.index might get really big
https://jira.talendforge.org/browse/TUP-27590

* fix(TUP-27590)Relationship.index might get really big
https://jira.talendforge.org/browse/TUP-27590
relatedItems is empty should not add  as a record

* fix(TUP-27590):Relationship.index might get really big
https://jira.talendforge.org/browse/TUP-27590

* fix(TUP-27590)Relationship.index might get really big
https://jira.talendforge.org/browse/TUP-27590
This commit is contained in:
Jane Ding
2020-07-23 18:40:41 +08:00
committed by GitHub
parent 8ae03ccad0
commit d152eb7b74
6 changed files with 148 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/handlers/JobAndNodesParametersRelationshipHandler.java
View File

@@ -117,6 +117,7 @@ public class JobAndNodesParametersRelationshipHandler implements IItemRelationsh
}
}
}
relationsMap.values().removeIf(value -> value.isEmpty());
return relationsMap;
}

6
main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/handlers/JobRoutinesItemRelationshipHandler.java
View File

@@ -14,6 +14,7 @@ package org.talend.core.repository.handlers;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.talend.core.model.properties.Item;
@@ -43,8 +44,13 @@ public class JobRoutinesItemRelationshipHandler extends AbstractJobItemRelations
Set<Relation> relationSet = new HashSet<Relation>();
if (processType.getParameters() != null && processType.getParameters().getRoutinesParameter() != null) {
Map<String, String> currentSystemRoutinesMap = RelationshipItemBuilder.getInstance().getCurrentSystemRoutinesMap();
for (Object o : processType.getParameters().getRoutinesParameter()) {
RoutinesParameterType itemInfor = (RoutinesParameterType) o;
if (currentSystemRoutinesMap.containsValue(itemInfor.getName())) {
// exclude system routines relation
continue;
}
Relation addedRelation = new Relation();
addedRelation.setId(itemInfor.getName());

3
main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/model/ProxyRepositoryFactory.java
View File

@@ -124,6 +124,7 @@ import org.talend.core.repository.constants.Constant;
import org.talend.core.repository.constants.FileConstants;
import org.talend.core.repository.i18n.Messages;
import org.talend.core.repository.recyclebin.RecycleBinManager;
import org.talend.core.repository.utils.ProjectDataJsonProvider;
import org.talend.core.repository.utils.RepositoryPathProvider;
import org.talend.core.repository.utils.XmiResourceManager;
import org.talend.core.runtime.CoreRuntimePlugin;
@@ -2083,6 +2084,8 @@ public final class ProxyRepositoryFactory implements IProxyRepositoryFactory {
ProjectManager.getInstance().getBeforeLogonRecords().clear();
ProjectManager.getInstance().getUpdatedRemoteHandlerRecords().clear();
ProjectDataJsonProvider.checkAndRectifyRelationShipSetting(project.getEmfProject());
// init dynamic distirbution after `beforeLogon`, before loading libraries.
initDynamicDistribution(monitor);

64
main/plugins/org.talend.core.repository/src/main/java/org/talend/core/repository/utils/ProjectDataJsonProvider.java
View File

@@ -22,6 +22,7 @@ import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -44,6 +45,7 @@ import org.talend.core.model.properties.Project;
import org.talend.core.model.properties.StatAndLogsSettings;
import org.talend.core.model.properties.Status;
import org.talend.core.model.properties.impl.PropertiesFactoryImpl;
import org.talend.core.model.relationship.RelationshipItemBuilder;
import org.talend.core.repository.constants.FileConstants;
import org.talend.core.repository.recyclebin.RecycleBinManager;
import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
@@ -217,6 +219,68 @@ public class ProjectDataJsonProvider {
}
}
public static void checkAndRectifyRelationShipSetting(Project project) throws PersistenceException {
File file = getSavingConfigurationFile(project.getTechnicalLabel(), FileConstants.RELATIONSHIP_FILE_NAME);
if (file == null || !file.exists()) {
return;
}
List<ItemRelationsJson> itemRelationsJsonsList = null;
TypeReference<List<ItemRelationsJson>> typeReference = new TypeReference<List<ItemRelationsJson>>() {
};
FileInputStream input = null;
try {
input = new FileInputStream(file);
itemRelationsJsonsList = new ObjectMapper().readValue(new FileInputStream(file), typeReference);
} catch (Exception e) {
throw new PersistenceException(e);
} finally {
closeInputStream(input);
}
if (itemRelationsJsonsList == null || itemRelationsJsonsList.isEmpty()) {
return;
}
Set<String> idVersionSet = new HashSet<String>();
List<ItemRelationsJson> relationJsonList = new ArrayList<ItemRelationsJson>();
Map<String, String> currentSystemRoutinesMap = RelationshipItemBuilder.getInstance().getCurrentSystemRoutinesMap();
boolean needModify = false;
for (ItemRelationsJson relationJson : itemRelationsJsonsList) {
ItemRelationJson baseItem = relationJson.getBaseItem();
String idversion = baseItem.getId() + ";" + baseItem.getVersion();
if (idVersionSet.contains(idversion)) {
// in case duplicate
needModify = true;
continue;
}
// remove system routines relation
int originalSize = relationJson.getRelatedItems().size();
relationJson.getRelatedItems()
.removeIf(relatedItem -> RelationshipItemBuilder.ROUTINE_RELATION.equals(relatedItem.getType())
&& currentSystemRoutinesMap.containsValue(relatedItem.getId()));
if (relationJson.getRelatedItems().size() != originalSize) {
needModify = true;
}
if (!relationJson.getRelatedItems().isEmpty()) {
relationJsonList.add(relationJson);
}
idVersionSet.add(idversion);
}
if (needModify) {
// re-load to project
if (relationJsonList != null && !relationJsonList.isEmpty()) {
project.getItemsRelations().clear();
for (ItemRelationsJson json : relationJsonList) {
project.getItemsRelations().add(json.toEmfObject());
}
}
// re-save relationship setting json file
saveRelationShips(project);
}
}
private static void loadMigrationTaskSetting(Project project, IPath projectFolderPath) throws PersistenceException {
File file = getLoadingConfigurationFile(projectFolderPath, FileConstants.MIGRATION_TASK_FILE_NAME);
if (file != null && file.exists()) {

25
main/plugins/org.talend.core.runtime/src/main/java/org/talend/core/model/relationship/RelationshipItemBuilder.java
View File

@@ -43,6 +43,7 @@ import org.talend.core.model.properties.ProcessItem;
import org.talend.core.model.properties.PropertiesFactory;
import org.talend.core.model.repository.ERepositoryObjectType;
import org.talend.core.model.repository.IRepositoryViewObject;
import org.talend.core.model.routines.RoutinesUtil;
import org.talend.core.runtime.CoreRuntimePlugin;
import org.talend.core.runtime.i18n.Messages;
import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
@@ -130,6 +131,8 @@ public class RelationshipItemBuilder {
private Map<Relation, Set<Relation>> referencesItemsRelations;
private Map<String, String> systemRoutinesMap;
private boolean loaded = false;
private boolean loading = false;
@@ -219,6 +222,28 @@ public class RelationshipItemBuilder {
this.proxyRepositoryFactory = proxyRepositoryFactory;
}
/**
*
* Current System Routines Map, key: routine id, value: routine name
*
* @return
*/
public Map<String, String> getCurrentSystemRoutinesMap() {
if (systemRoutinesMap != null && !systemRoutinesMap.isEmpty()) {
return systemRoutinesMap;
}
if (systemRoutinesMap == null) {
systemRoutinesMap = new HashMap<String, String>();
}
List<IRepositoryViewObject> currentSystemRoutines = RoutinesUtil.getCurrentSystemRoutines();
for (IRepositoryViewObject object : currentSystemRoutines) {
systemRoutinesMap.put(object.getProperty().getId(), object.getProperty().getLabel());
}
return systemRoutinesMap;
}
/**
* Look for every linked items who use the selected id, no matter the version. Usefull when want to delete an item
* since it will delete every versions.

60
test/plugins/org.talend.core.repository.test/src/org/talend/core/repository/handlers/JobAndNodesParametersRelationshipHandlerTest.java
View File

@@ -12,6 +12,10 @@
// ============================================================================
package org.talend.core.repository.handlers;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.Map;
import java.util.Set;
@@ -25,13 +29,14 @@ import org.talend.core.model.properties.PropertiesFactory;
import org.talend.core.model.properties.Property;
import org.talend.core.model.relationship.Relation;
import org.talend.core.model.relationship.RelationshipItemBuilder;
import org.talend.core.repository.model.ProxyRepositoryFactory;
import org.talend.designer.core.model.utils.emf.talendfile.ElementParameterType;
import org.talend.designer.core.model.utils.emf.talendfile.NodeType;
import org.talend.designer.core.model.utils.emf.talendfile.ProcessType;
import org.talend.designer.core.model.utils.emf.talendfile.TalendFileFactory;
import org.talend.designer.joblet.model.JobletFactory;
import org.talend.designer.joblet.model.JobletProcess;
import static org.mockito.Mockito.*;
/**
* DOC ggu class global comment. Detailled comment
*/
@@ -114,16 +119,49 @@ public class JobAndNodesParametersRelationshipHandlerTest {
Map<Relation, Set<Relation>> relations = handler.find(item);
Assert.assertNotNull(relations);
Assert.assertTrue(relations.size() > 0);
Assert.assertTrue(relations.values().isEmpty());
Relation processRelation = relations.keySet().iterator().next();
Assert.assertNotNull(processRelation);
Assert.assertEquals(AbstractProcessItemRelationshipHandlerTest.ITEM_ID, processRelation.getId());
Assert.assertEquals("0.1", processRelation.getVersion());
Assert.assertEquals(RelationshipItemBuilder.JOB_RELATION, processRelation.getType());
}
Set<Relation> set = relations.get(processRelation);
Assert.assertNotNull(set);
Assert.assertTrue(set.isEmpty()); // no relations
@Test
public void testFindParameters() {
ProcessItem item = PropertiesFactory.eINSTANCE.createProcessItem();
Property property = PropertiesFactory.eINSTANCE.createProperty();
property.setId(AbstractProcessItemRelationshipHandlerTest.ITEM_ID);
property.setVersion("0.1");
property.setLabel("test");
item.setProperty(property);
ProcessType process = TalendFileFactory.eINSTANCE.createProcessType();
item.setProcess(process);
NodeType node = TalendFileFactory.eINSTANCE.createNodeType();
node.setComponentName("tRunJob");
ElementParameterType param = TalendFileFactory.eINSTANCE.createElementParameterType();
param.setName("PROCESS:PROCESS_TYPE_VERSION");
param.setValue(RelationshipItemBuilder.LATEST_VERSION);
String relatedId = "project" + ":" + ProxyRepositoryFactory.getInstance().getNextId();
ElementParameterType param1 = TalendFileFactory.eINSTANCE.createElementParameterType();
param1.setName("PROCESS:PROCESS_TYPE_PROCESS");
param1.setValue(relatedId);
node.getElementParameter().add(param);
node.getElementParameter().add(param1);
process.getNode().add(node);
Map<Relation, Set<Relation>> relationsMap = handler.find(item);
Assert.assertNotNull(relationsMap);
Assert.assertTrue(relationsMap.values().size() > 0);
Relation baseRelation = relationsMap.keySet().iterator().next();
Assert.assertNotNull(baseRelation);
Assert.assertEquals(property.getId(), baseRelation.getId());
Assert.assertEquals(property.getVersion(), baseRelation.getVersion());
Assert.assertEquals(RelationshipItemBuilder.JOB_RELATION, baseRelation.getType());
Set<Relation> relatedRelations = relationsMap.get(baseRelation);
Assert.assertNotNull(relatedRelations);
Assert.assertTrue(relatedRelations.size() > 0);
Relation relatedRelation = relatedRelations.iterator().next();
Assert.assertEquals(relatedId, relatedRelation.getId());
Assert.assertEquals(RelationshipItemBuilder.LATEST_VERSION, relatedRelation.getVersion());
Assert.assertEquals(RelationshipItemBuilder.JOB_RELATION, relatedRelation.getType());
}
}