Compare commits

...

1 Commits

Author SHA1 Message Date
kjwang
34c22ddeee Fix TUP-41447 TCK list migration (#9466) (#9546)
Fix TUP-41447 TCK list migration
https://jira.talendforge.org/browse/TUP-41447
2024-01-31 14:31:54 +08:00
7 changed files with 402 additions and 41 deletions

View File

@@ -318,6 +318,15 @@
name="UpdateEncryptTckMetadataMigrationTask"
version="8.0.1">
</projecttask>
<projecttask
beforeLogon="false"
breaks="8.0.0"
class="org.talend.sdk.component.studio.metadata.migration.UpgradeTacokitMetadataMigrationTask"
description="Migrate tacokit metadata"
id="org.talend.sdk.component.studio.metadata.migration.UpgradeTacokitMetadataMigrationTask"
name="UpgradeTacokitMetadataMigrationTask"
version="8.0.1">
</projecttask>
<projecttask
beforeLogon="false"
breaks="8.0.1"

View File

@@ -12,9 +12,14 @@
*/
package org.talend.sdk.component.studio.metadata.migration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
@@ -37,6 +42,7 @@ import org.talend.sdk.component.studio.exception.UserCancelledException;
import org.talend.sdk.component.studio.i18n.Messages;
import org.talend.sdk.component.studio.metadata.TaCoKitCache;
import org.talend.sdk.component.studio.metadata.model.TaCoKitConfigurationModel;
import org.talend.sdk.component.studio.model.parameter.ValueConverter;
import org.talend.sdk.component.studio.model.update.TaCoKitUpdateManager;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1Component;
@@ -47,7 +53,6 @@ import org.talend.sdk.studio.process.TaCoKitNode;
* DOC cmeng class global comment. Detailled comment
*/
public class TaCoKitMigrationManager {
private V1ConfigurationType configurationClient;
private final V1Component componentClient = Lookups.client().v1().component();
@@ -56,7 +61,8 @@ public class TaCoKitMigrationManager {
configurationClient = Lookups.client().v1().configurationType();
}
public void checkProcessItemMigration(final Item item, final String compType, final IProgressMonitor progressMonitor) throws UserCancelledException {
public void checkProcessItemMigration(final Item
item, final String compType, final IProgressMonitor progressMonitor) throws UserCancelledException {
IProgressMonitor monitor = progressMonitor;
if (monitor == null) {
monitor = new NullProgressMonitor();
@@ -179,11 +185,72 @@ public class TaCoKitMigrationManager {
final int storedVersion = configModel.getVersion();
final int newVersion = configModel.getConfigurationVersion();
monitor.subTask(Messages.getString("migration.check.progress.execute", label, storedVersion, newVersion)); //$NON-NLS-1$
Map<String, String> migratedProperties = configurationClient.migrate(configModel.getConfigurationId(),
configModel.getVersion(), configModel.getProperties());
configModel.migrate(migratedProperties);
configModel.getVersion(), expandTableParameters(configModel));
configModel.migrate(collapseTableParameters(configModel, migratedProperties));
}
private Map<String, String> expandTableParameters(TaCoKitConfigurationModel configModel) {
Map<String, String> properties = new HashMap<>();
for (String key : configModel.getProperties().keySet()) {
String value = configModel.getProperties().get(key);
boolean isAdded = false;
if (ValueConverter.isListParameterValue(value)) {
List<Map<String, Object>> listValue = ValueConverter.toTable(value);
if (listValue.size() > 0) {
for (int i = 0; i < listValue.size(); i++) {
Map<String, Object> map = listValue.get(i);
for (String name : map.keySet()) {
String nameWithIndex = ValueConverter.getTableParameterNameWithIndex(i, name);
properties.put(nameWithIndex, map.get(name).toString());
}
}
isAdded = true;
}
}
if (!isAdded) {
properties.put(key, value);
}
}
return properties;
}
private Map<String, String> collapseTableParameters(TaCoKitConfigurationModel configModel,
Map<String, String> migratedProperties) {
Map<String, String> properties = new LinkedHashMap<String, String>();
Set<String> processedName = new HashSet<String>();
for (String key : migratedProperties.keySet()) {
int paramIndex = ValueConverter.getTableParameterIndex(key);
if (paramIndex >= 0) {
String paramName = ValueConverter.getMainTableParameterName(key);
if (processedName.contains(paramName)) {
continue;
} else {
processedName.add(paramName);
}
Map<String, String> newParams = ValueConverter.getSameNameTableParameter(paramName, migratedProperties);
List<Map<String, String>> newProperties = new ArrayList<Map<String, String>>();
String firstKey = null;
Map<String, String> data = null;
for (String newParamName : newParams.keySet()) {
String propertyName = ValueConverter.getTableParameterNameNoIndex(newParamName);
if (firstKey == null) {
firstKey = propertyName;
}
if (firstKey.equals(propertyName)) {
data = new HashMap<String, String>();
newProperties.add(data);
}
data.put(propertyName, newParams.get(newParamName));
}
properties.put(paramName, ValueConverter.toStringValue(newProperties));
} else {
properties.put(key, migratedProperties.get(key));
}
}
return properties;
}
public void updatedRelatedItems(final ConnectionItem item, final String version, final IProgressMonitor progressMonitor) {
IProgressMonitor monitor = progressMonitor;

View File

@@ -0,0 +1,74 @@
// ============================================================================
//
// Copyright (C) 2006-2024 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.sdk.component.studio.metadata.migration;
import java.util.Date;
import java.util.GregorianCalendar;
import org.talend.commons.exception.ExceptionHandler;
import org.talend.commons.runtime.service.ITaCoKitService;
import org.talend.core.model.metadata.builder.connection.Connection;
import org.talend.core.model.migration.AbstractItemMigrationTask;
import org.talend.core.model.properties.ConnectionItem;
import org.talend.core.model.properties.Item;
import org.talend.core.repository.model.ProxyRepositoryFactory;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.metadata.model.TaCoKitConfigurationModel;
public class UpgradeTacokitMetadataMigrationTask extends AbstractItemMigrationTask {
private ProxyRepositoryFactory factory = ProxyRepositoryFactory.getInstance();
@Override
public Date getOrder() {
GregorianCalendar gc = new GregorianCalendar(2024, 01, 23, 12, 0, 0);
return gc.getTime();
}
@Override
public ExecutionResult execute(Item item) {
boolean modified = false;
if (item instanceof ConnectionItem) {
try {
ConnectionItem conItem = (ConnectionItem) item;
Connection connection = conItem.getConnection();
if (!TaCoKitConfigurationModel.isTacokit(connection)) {
return ExecutionResult.NOTHING_TO_DO;
}
try {
ITaCoKitService.getInstance().waitForStart();
} catch (Throwable t) {
// don't block if fail
ExceptionHandler.process(t);
}
TaCoKitMigrationManager taCoKitMigrationManager = Lookups.taCoKitCache().getMigrationManager();
TaCoKitConfigurationModel configModel = new TaCoKitConfigurationModel(conItem.getConnection());
if (configModel.needsMigration()) {
taCoKitMigrationManager.migrate(configModel, null);
factory.save(item, true);
modified = true;
}
} catch (IllegalArgumentException e) {
ExceptionHandler.process(e);
} catch (Exception e) {
ExceptionHandler.process(e);
return ExecutionResult.FAILURE;
}
}
if (modified) {
return ExecutionResult.SUCCESS_NO_ALERT;
} else {
return ExecutionResult.NOTHING_TO_DO;
}
}
}

View File

@@ -16,9 +16,12 @@
package org.talend.sdk.component.studio.model.parameter;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Pattern;
/**
@@ -79,6 +82,16 @@ public final class ValueConverter {
}
return table;
}
/**
* The logic same with TableElementParameter.getStringValue
* @param list
* @return
*/
public static String toStringValue(List<Map<String, String>> list) {
return list.toString();
}
/**
* Checks whether String representation of the list is empty or not
@@ -110,4 +123,79 @@ public final class ValueConverter {
return CURLY_BRACKETS_PATTERN.matcher(str).replaceAll("");
}
public static String getMainTableParameterName(String name) {
int begin = name.indexOf("[");
int end = name.indexOf("]");
if (begin > 0 && end > 0 && end > begin) {
return name.substring(0, begin);
}
return name;
}
public static int getTableParameterIndex(String name) {
int begin = name.indexOf("[");
int end = name.indexOf("]");
if (begin > 0 && end > 0 && end > begin) {
return Integer.parseInt(name.substring(begin + 1, end));
}
return -1;
}
public static String getTableParameterNameNoIndex(String name) {
int begin = name.indexOf("[");
int end = name.indexOf("]");
if (begin > 0 && end > 0 && end > begin) {
return name.substring(0, begin + 1) + name.substring(end);
}
return name;
}
public static String getTableParameterNameWithIndex(int index, String paramName) {
if (paramName != null && paramName.indexOf("[") >= 0 && paramName.indexOf("]") > 0) {
StringBuffer sb = new StringBuffer();
sb.append(paramName.substring(0, paramName.indexOf("[") + 1)).append(index)
.append(paramName.substring(paramName.indexOf("]")));
return sb.toString();
}
return null;
}
/**
* Get same main name parameters that key sorted by index
* @param paramName
* @param migratedProperties
* @return same main name parameters that key sorted by index
*/
public static Map<String, String> getSameNameTableParameter(String paramName, Map<String, String> migratedProperties) {
Map<String, String> properties = new HashMap<String, String>();
for (String key : migratedProperties.keySet()) {
String name = ValueConverter.getMainTableParameterName(key);
if (paramName.equals(name)) {
properties.put(key, migratedProperties.get(key));
}
}
Map<String, String> sortedMap = new TreeMap<String, String>(new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
int index1 = ValueConverter.getTableParameterIndex(o1);
int index2 = ValueConverter.getTableParameterIndex(o2);
if (index1 != index2) {
return index1 - index2;
}
return o1.compareTo(o2);
}
});
sortedMap.putAll(properties);
return sortedMap;
}
public static boolean isListParameterValue(String value) {
if (value != null && value.trim().startsWith("[") && value.trim().endsWith("]")) {
return true;
}
return false;
}
}

View File

@@ -139,6 +139,8 @@ public class TaCoKitContextHandler extends AbstractRepositoryContextHandler {
List<Map<String, Object>> tableValueList = ValueConverter.toTable((String) valueModel.getValue());
List<Map<String, Object>> originalTableValueList = new ArrayList<>();
for (Map<String, Object> map : tableValueList) {
Map<String, Object> originMap = new HashMap<String, Object>();
originalTableValueList.add(originMap);
for (Entry<String, Object> entryTable : map.entrySet()) {
Object value = entryTable.getValue();
if (value instanceof String) {
@@ -146,8 +148,6 @@ public class TaCoKitContextHandler extends AbstractRepositoryContextHandler {
if (tableOriginalValue != null) {
String[] splitValues = tableOriginalValue.split(";");
for (String s : splitValues) {
Map<String, Object> originMap = new HashMap<String, Object>();
originalTableValueList.add(originMap);
originMap.put(entryTable.getKey(), TalendQuoteUtils.removeQuotes(s));
}
}

View File

@@ -12,7 +12,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import org.eclipse.emf.common.util.EList;
@@ -28,6 +27,7 @@ import org.talend.sdk.component.server.front.model.ConfigTypeNode;
import org.talend.sdk.component.server.front.model.SimplePropertyDefinition;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.model.parameter.PropertyDefinitionDecorator;
import org.talend.sdk.component.studio.model.parameter.ValueConverter;
import org.talend.sdk.component.studio.model.parameter.VersionParameter;
import org.talend.sdk.component.studio.model.parameter.WidgetTypeMapper;
import org.talend.sdk.component.studio.util.TaCoKitConst;
@@ -136,7 +136,7 @@ public final class TaCoKitNode {
} else if (firstColumnKey.equals(eValue.getElementRef())){
index++;
}
String paramName = getTableParamName(index, eValue);
String paramName = ValueConverter.getTableParameterNameWithIndex(index, eValue.getElementRef());
if (paramName != null) {
properties.put(paramName, eValue.getValue());
}
@@ -145,17 +145,6 @@ public final class TaCoKitNode {
}
}
private String getTableParamName(int index, ElementValueType elementValueType) {
String paramValue = elementValueType.getElementRef();
if (paramValue != null && paramValue.indexOf("[") >= 0 && paramValue.indexOf("]") > 0) {
StringBuffer sb = new StringBuffer();
sb.append(paramValue.substring(0, paramValue.indexOf("[") + 1)).append(index)
.append(paramValue.substring(paramValue.indexOf("]")));
return sb.toString();
}
return null;
}
private boolean isComponentProperty(Collection<SimplePropertyDefinition> props, final String name) {
return props.stream().anyMatch(property -> property.getPath().equals(name));
}
@@ -188,27 +177,30 @@ public final class TaCoKitNode {
}
private void fillTableParamData(List<ElementParameterType> tableFieldParamList, String paramKey, String paramValue) {
String paramName = paramKey.substring(0, paramKey.indexOf("["));
String elemRef = paramKey.substring(0, paramKey.indexOf("[") + 1) + paramKey.substring(paramKey.indexOf("]"));
int paramIndex = Integer.parseInt(paramKey.substring(paramKey.indexOf("[") + 1, paramKey.indexOf("]")));
String paramName = ValueConverter.getMainTableParameterName(paramKey);
String elemRef = ValueConverter.getTableParameterNameNoIndex(paramKey);
int paramIndex = ValueConverter.getTableParameterIndex(paramKey);
ElementParameterType sameNameParam = null;
for (ElementParameterType param : tableFieldParamList) {
if (param.getName().equals(paramName)) {
sameNameParam = param;
List list = param.getElementValue();
int index = 0;
int rowIndex = -1;
String firstKey = null;
for (int i = 0; i < list.size(); i++) {
ElementValueType eValue = (ElementValueType) list.get(i);
if (elemRef.equals(eValue.getElementRef())) {
if (paramIndex == index) {
eValue.setValue(paramValue);
return;
} else {
index ++;
}
if (firstKey == null) {
firstKey = eValue.getElementRef();
}
if (firstKey.equals(eValue.getElementRef())) {
rowIndex++;
}
if (elemRef.equals(eValue.getElementRef()) && paramIndex == rowIndex) {
eValue.setValue(paramValue);
return;
}
}
}
}
}
if (sameNameParam == null) {
sameNameParam = TalendFileFactoryImpl.eINSTANCE.createElementParameterType();
@@ -219,7 +211,28 @@ public final class TaCoKitNode {
ElementValueType elementValueType = TalendFileFactoryImpl.eINSTANCE.createElementValueType();
elementValueType.setElementRef(elemRef);
elementValueType.setValue(paramValue);
sameNameParam.getElementValue().add(elementValueType);
boolean isAdded = false;
if (sameNameParam.getElementValue().size() > 0) {
int rowIndex = -1;
String firstKey = null;
for (int insertIndex = 0; insertIndex < sameNameParam.getElementValue().size(); insertIndex++) {
ElementValueType e = (ElementValueType) sameNameParam.getElementValue().get(insertIndex);
if (firstKey == null) {
firstKey = e.getElementRef();
}
if (firstKey.equals(e.getElementRef())) {
rowIndex++;
}
if (rowIndex > paramIndex) {
sameNameParam.getElementValue().add(insertIndex, elementValueType);
isAdded = true;
break;
}
}
}
if (!isAdded) {
sameNameParam.getElementValue().add(elementValueType);
}
}
private ElementParameterType createParameter(final String name, final String value) {

View File

@@ -1,21 +1,21 @@
/**
* Copyright (C) 2006-2021 Talend Inc. - www.talend.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.talend.sdk.component.studio.model.parameter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.HashMap;
@@ -59,4 +59,114 @@ public class ValueConverterTest {
assertEquals(empty, actual);
}
@Test
public void testGetMainTableParameterName() {
String paramName = "configuration.headers[].key";
assertEquals("configuration.headers", ValueConverter.getMainTableParameterName(paramName));
paramName = "configuration.headers[0].key";
assertEquals("configuration.headers", ValueConverter.getMainTableParameterName(paramName));
paramName = "configuration.headers[0]";
assertEquals("configuration.headers", ValueConverter.getMainTableParameterName(paramName));
}
@Test
public void testGetTableParameterIndex() {
String paramName = "configuration.headers[0].key";
assertEquals(0, ValueConverter.getTableParameterIndex(paramName));
paramName = "configuration.headers[1].key";
assertEquals(1, ValueConverter.getTableParameterIndex(paramName));
paramName = "configuration.headers";
assertEquals(-1, ValueConverter.getTableParameterIndex(paramName));
}
@Test
public void testGetTableParameterNameNoIndex() {
String paramName = "configuration.headers[0].key";
assertEquals("configuration.headers[].key", ValueConverter.getTableParameterNameNoIndex(paramName));
paramName = "configuration.headers[1].key";
assertEquals("configuration.headers[].key", ValueConverter.getTableParameterNameNoIndex(paramName));
paramName = "configuration.headers[0].value";
assertEquals("configuration.headers[].value", ValueConverter.getTableParameterNameNoIndex(paramName));
}
@Test
public void testGetTableParameterNameWithIndex() {
String paramName = "configuration.headers[].key";
assertEquals("configuration.headers[0].key", ValueConverter.getTableParameterNameWithIndex(0, paramName));
assertEquals("configuration.headers[1].key", ValueConverter.getTableParameterNameWithIndex(1, paramName));
paramName = "configuration.headers";
assertNull(ValueConverter.getTableParameterNameWithIndex(1, paramName));
}
@Test
public void testGetSameNameTableParameter() {
Map<String, String> testData = new HashMap<String, String>();
testData.put("configuration.headers[0].value", "value-0");
testData.put("configuration.headers[2].query", "MAIN");
testData.put("configuration.headers[1].query", "MAIN");
testData.put("configuration.headers[0].query", "MAIN");
testData.put("configuration.headers[2].key", "h3");
testData.put("configuration.headers[1].key", "h2");
testData.put("configuration.headers[0].key", "h1");
testData.put("configuration.headers[1].value", "value-1");
testData.put("configuration.headers[2].value", "value-2");
testData.put("configuration.datastore.authentication.oauth20.params[2]", "scope2");
testData.put("configuration.datastore.authentication.oauth20.params[1]", "scope1");
testData.put("configuration.datastore.authentication.oauth20.params[0]", "scope0");
Map<String, String> sameNameParms = ValueConverter.getSameNameTableParameter("configuration.headers", testData);
assertEquals(9, sameNameParms.size());
boolean hasError = false;
int lastIndex = 0;
for (String key : sameNameParms.keySet()) {
int index = ValueConverter.getTableParameterIndex(key);
if (index >= lastIndex) {
lastIndex = index;
} else {
hasError = true;
}
}
assertFalse(hasError);
sameNameParms = ValueConverter.getSameNameTableParameter("configuration.datastore.authentication.oauth20.params",
testData);
assertEquals(3, sameNameParms.size());
lastIndex = 0;
for (String key : sameNameParms.keySet()) {
int index = ValueConverter.getTableParameterIndex(key);
if (index >= lastIndex) {
lastIndex = index;
} else {
hasError = true;
}
}
assertFalse(hasError);
}
@Test
public void testIsListParameterValue() {
String data = "[]";
assertTrue(ValueConverter.isListParameterValue(data));
data = " [ ] ";
assertTrue(ValueConverter.isListParameterValue(data));
data = "[{configuration.headers[].key=\"h1\", configuration.headers[].value=\"11\"}, {configuration.headers[].key=\"h2\", configuration.headers[].value=\"22\"}, {configuration.headers[].key=\"h3\", configuration.headers[].value=\"33\"}]";
assertTrue(ValueConverter.isListParameterValue(data));
data = "[{configuration.headers[].key=\"h1\", configuration.headers[].value=\"11\"}]";
assertTrue(ValueConverter.isListParameterValue(data));
data = "[{configuration.headers[].key=\"h1\", configuration.headers[].value=\"11\"}";
assertFalse(ValueConverter.isListParameterValue(data));
}
}