Compare commits

...

1 Commits

Author SHA1 Message Date
Christophe Le Saec
c2fc7e5862 TDI-46926 : join dependencies 2022-02-03 16:44:25 +01:00
27 changed files with 1935 additions and 599 deletions

View File

@@ -14,6 +14,7 @@
<description>Studio integration of the Talend Component Kit framework.</description>
<properties>
<component-runtime.version>1.42.0-SNAPSHOT</component-runtime.version>
<commons-lang3.version>3.11</commons-lang3.version>
<mockito.version>2.23.0</mockito.version>
<oro.version>2.0.8</oro.version>
@@ -323,4 +324,4 @@
</dependencies>
</profile>
</profiles>
</project>
</project>

View File

@@ -1,12 +1,12 @@
/**
* Copyright (C) 2006-2021 Talend Inc. - www.talend.com
*
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,19 +21,9 @@ import static org.talend.sdk.component.studio.model.ReturnVariables.AFTER;
import static org.talend.sdk.component.studio.model.ReturnVariables.RETURN_ERROR_MESSAGE;
import static org.talend.sdk.component.studio.model.ReturnVariables.RETURN_TOTAL_RECORD_COUNT;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.eclipse.jface.resource.ImageDescriptor;
import org.talend.commons.CommonsPlugin;
@@ -67,12 +57,13 @@ import org.talend.sdk.component.server.front.model.ComponentId;
import org.talend.sdk.component.server.front.model.ComponentIndex;
import org.talend.sdk.component.server.front.model.ConfigTypeNodes;
import org.talend.sdk.component.server.front.model.SimplePropertyDefinition;
import org.talend.sdk.component.studio.dependencies.ComponentReference;
import org.talend.sdk.component.studio.dependencies.ComponentReferenceFinder;
import org.talend.sdk.component.studio.enums.ETaCoKitComponentType;
import org.talend.sdk.component.studio.metadata.migration.TaCoKitMigrationManager;
import org.talend.sdk.component.studio.model.connector.ConnectorCreatorFactory;
import org.talend.sdk.component.studio.model.connector.TaCoKitNodeConnector;
import org.talend.sdk.component.studio.model.parameter.ElementParameterCreator;
import org.talend.sdk.component.studio.model.parameter.Metadatas;
import org.talend.sdk.component.studio.model.parameter.*;
import org.talend.sdk.component.studio.mvn.Mvn;
import org.talend.sdk.component.studio.service.ComponentService;
import org.talend.sdk.component.studio.util.TaCoKitConst;
@@ -112,10 +103,12 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
private final ConfigTypeNodes configTypeNodes;
protected final TaCoKitMigrationManager manager = Lookups.taCoKitCache().getMigrationManager();
public ComponentModel(final ComponentIndex component, final ComponentDetail detail, final ConfigTypeNodes configTypeNodes, final ImageDescriptor image32,
final String reportPath, final boolean isCatcherAvailable) {
final String reportPath, final boolean isCatcherAvailable) {
setPaletteType(ComponentCategory.CATEGORY_4_DI.getName());
this.index = component;
this.detail = detail;
@@ -135,7 +128,8 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
}
}
@Deprecated // to drop since it is not used at all in main code
@Deprecated
// to drop since it is not used at all in main code
ComponentModel(final ComponentIndex component, final ComponentDetail detail) {
setPaletteType("DI");
this.index = component;
@@ -330,7 +324,7 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
errorMessage.setAvailability(AFTER);
returnVariables.add(errorMessage);
if(!ETaCoKitComponentType.standalone.equals(getTaCoKitComponentType())) {
if (!ETaCoKitComponentType.standalone.equals(getTaCoKitComponentType())) {
NodeReturn numberLinesMessage = new NodeReturn();
numberLinesMessage.setType(JavaTypesManager.INTEGER.getId());
numberLinesMessage.setDisplayName(ComponentReturnVariableUtils.getTranslationForVariable(RETURN_TOTAL_RECORD_COUNT,
@@ -409,10 +403,11 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
*/
@Override
public List<ModuleNeeded> getModulesNeeded(final INode iNode) {
final ComponentService.Dependencies dependencies = getDependencies();
if (modulesNeeded == null) {
synchronized (this) {
if (modulesNeeded == null) {
final ComponentService.Dependencies dependencies = getDependencies();
modulesNeeded = new LinkedHashSet<>(20);
modulesNeeded.addAll(dependencies
.getCommon()
@@ -431,32 +426,8 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
}
}
final Map<String, ?> componentDependencies = !Lookups.configuration().isActive() ? null : Lookups.client().v1().component().dependencies(detail.getId().getId());
if (componentDependencies != null && componentDependencies.containsKey("dependencies")) {
final Collection<String> coordinates = Collection.class.cast(Map.class
.cast(Map.class.cast(componentDependencies.get("dependencies")).values().iterator().next())
.get("dependencies"));
if (coordinates != null) {
modulesNeeded.addAll(coordinates.stream()
.map(coordinate -> new ModuleNeeded(getName(), "", true, Mvn.locationToMvn(coordinate).replace(MavenConstants.LOCAL_RESOLUTION_URL + '!', "")))
.collect(Collectors.toList()));
//TODO fix this, this is wrong here as coordinates is a list object, not string, it's on purpose?
if (coordinates.contains("org.apache.beam") || coordinates.contains(":beam-sdks-java-io")) {
modulesNeeded.addAll(dependencies
.getBeam()
.stream()
.map(s -> new ModuleNeeded(getName(), "", true, s))
.collect(toList()));
}
String content = coordinates.toString();
if(content.contains("org.scala-lang") && !content.contains(":scala-library:")) {
//we can't add this dependency to connector as spark/beam class conflict for TPD, so add here as provided by platform like spark/beam
modulesNeeded.add(new ModuleNeeded(getName(), "", true, "mvn:org.scala-lang/scala-library/2.12.12"));
}
}
}
final List<ModuleNeeded> componentsDeps = this.componentDependencies(dependencies, detail.getId());
modulesNeeded.addAll(componentsDeps);
// We're assuming that pluginLocation has format of groupId:artifactId:version
final String location = index.getId().getPluginLocation().trim();
@@ -464,7 +435,126 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
}
}
}
return new ArrayList<>(modulesNeeded);
final List<ModuleNeeded> modules = new ArrayList<>(modulesNeeded);
if (iNode != null) {
// Add dependencies of a connector use by this connector
final List<PropertyNode> connectors = iNode.getElementParameters()
.stream()
.filter(TaCoKitElementParameter.class::isInstance)
.map(TaCoKitElementParameter.class::cast)
.map(this::extractNodeWithDependencies) // contains a dependency to another tck connector.
.filter(Objects::nonNull)
.distinct()
.collect(toList());
connectors.stream()
.flatMap((PropertyNode p) -> this.extractComponent(p, iNode)) // family name to Component Detail
.filter(Objects::nonNull)
.map((final ComponentDetail connectorDetails) -> this.connectorDependencies(dependencies, connectorDetails.getId()))
.forEach(modules::addAll);
}
return modules;
}
private Stream<ComponentDetail> extractComponent(final PropertyNode property, final INode node) {
return this.extractComponentRef(property, node)
.map((final ComponentReference ref) -> {
final String connectorRef = ref.getFamily() + ref.getName();
return Lookups.service().getDetail(connectorRef).orElse(null);
});
}
private Stream<ComponentReference> extractComponentRef(final PropertyNode property, final INode node) {
return ComponentReferenceFinder.getFinder(property)
.find(property, node);
}
/**
* Metadata "dependencies::connector" means that a connector is a dependency of another.
* dependencies should contains dependencies of this connector plus the connector itself.
* @param dependencies : additional dependencies if needed (beam ...)
* @param componentId : id of dependency connector.
* @return all dependencies.
*/
private List<ModuleNeeded> connectorDependencies(final ComponentService.Dependencies dependencies,
final ComponentId componentId) {
// sub dependencies of connector.
final List<ModuleNeeded> modules = new ArrayList<>(30);
modules.addAll(this.componentDependencies(dependencies, componentId));
// connectors itself.
final String componentMavenLocation = Mvn.locationToMvn(componentId.getPluginLocation());
final ModuleNeeded connectorDep = this.moduleDependency(componentMavenLocation);
modules.add(connectorDep);
return modules;
}
private List<ModuleNeeded> componentDependencies(final ComponentService.Dependencies dependencies,
final ComponentId componentId) {
final Map<String, ?> componentDependencies = !Lookups.configuration().isActive() ? null : Lookups.client().v1().component().dependencies(componentId.getId());
if (componentDependencies != null && componentDependencies.containsKey("dependencies")) {
final Collection<String> coordinates = Collection.class.cast(Map.class
.cast(Map.class.cast(componentDependencies.get("dependencies")).values().iterator().next())
.get("dependencies"));
if (coordinates != null) {
final Stream<String> directDependencies = coordinates.stream()
.map(Mvn::locationToMvn)
.map((String gav) -> gav.replace(MavenConstants.LOCAL_RESOLUTION_URL + '!', ""));
//TODO fix this, this is wrong here as coordinates is a list object, not string, it's on purpose?
final Stream<String> beamDependencies;
if (coordinates.contains("org.apache.beam") || coordinates.contains(":beam-sdks-java-io")) {
beamDependencies = dependencies.getBeam().stream();
} else {
beamDependencies = Stream.empty();
}
final Stream<String> scalaDependencies;
String content = coordinates.toString();
if (content.contains("org.scala-lang") && !content.contains(":scala-library:")) {
//we can't add this dependency to connector as spark/beam class conflict for TPD, so add here as provided by platform like spark/beam
scalaDependencies = Stream.of("mvn:org.scala-lang/scala-library/2.12.12");
} else {
scalaDependencies = Stream.empty();
}
return Stream.concat(Stream.concat(directDependencies, beamDependencies), scalaDependencies)
.map(this::moduleDependency)
.collect(Collectors.toList());
}
}
return Collections.emptyList();
}
private ModuleNeeded moduleDependency(final String gav) {
return new ModuleNeeded(getName(), "", true, gav);
}
private PropertyNode extractNodeWithDependencies(final TaCoKitElementParameter parameter) {
final Optional<PropertyNode> propertyNode;
if (parameter instanceof TableElementParameter) {
propertyNode = Optional.ofNullable(parameter)
.map(TaCoKitElementParameter::getPropertyNode);
}
else {
propertyNode = Optional.ofNullable(parameter)
.map(TaCoKitElementParameter::getPropertyNode)
.map(PropertyNode::getParent);
}
Optional<Map<String, String>> metadata = propertyNode
.map(PropertyNode::getProperty)
.map(PropertyDefinitionDecorator::getMetadata);
if (metadata != null
&& metadata.isPresent()
&& metadata.get().containsKey("dependencies::connector")) {
return propertyNode.get();
}
return null;
}
protected boolean hasTcomp0Component(final INode iNode) {
@@ -605,11 +695,11 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
}
try {
switch (event) {
case IConnection.EVENT_UPDATE_INPUT_CONNECTION:
onUpdateInputConnection(parameters);
break;
default:
break;
case IConnection.EVENT_UPDATE_INPUT_CONNECTION:
onUpdateInputConnection(parameters);
break;
default:
break;
}
} catch (Exception e) {
ExceptionHandler.process(e);
@@ -726,7 +816,7 @@ public class ComponentModel extends AbstractBasicComponent implements IAdditiona
return configTypeNodes;
}
public ComponentId getId(){
public ComponentId getId() {
return this.index.getId();
}

View File

@@ -45,7 +45,7 @@ import org.talend.sdk.component.studio.service.UiActionsThreadPool;
import org.talend.sdk.component.studio.ui.composite.TaCoKitComposite;
import org.talend.sdk.component.studio.ui.composite.problemmanager.ComponentViewProblemManager;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
public final class Lookups {
@@ -122,8 +122,8 @@ public final class Lookups {
}
}
public static WebSocketClient client() {
return lookup(WebSocketClient.class);
public static ServicesClient client() {
return lookup(ServicesClient.class);
}
public static ComponentService service() {
@@ -146,6 +146,9 @@ public final class Lookups {
private static <T> T lookup(final Class<T> type) {
final BundleContext context = Platform.getBundle(TaCoKitConst.BUNDLE_ID).getBundleContext();
final ServiceReference<T> clientRef = context.getServiceReference(type);
if (clientRef == null) {
throw new IllegalArgumentException("Can't find service reference for class '" + type.getName() + "'");
}
return context.getService(clientRef);
}

View File

@@ -25,11 +25,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.log4j.Level;
import org.apache.tomcat.websocket.Constants;
import org.osgi.framework.BundleContext;
import org.osgi.framework.ServiceReference;
import org.osgi.framework.ServiceRegistration;
import org.talend.commons.CommonsPlugin;
import org.talend.commons.exception.CommonExceptionHandler;
import org.talend.commons.exception.ExceptionHandler;
import org.talend.commons.utils.VersionUtils;
import org.talend.core.nexus.TalendMavenResolver;
@@ -42,7 +44,8 @@ import org.talend.sdk.component.studio.service.ComponentService;
import org.talend.sdk.component.studio.service.Configuration;
import org.talend.sdk.component.studio.service.UiActionsThreadPool;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
import org.talend.sdk.component.studio.websocket.WebSocketClientImpl;
public class ServerManager {
@@ -52,7 +55,9 @@ public class ServerManager {
private final Collection<ServiceRegistration<?>> services = new ArrayList<>();
private WebSocketClient client;
private ServicesClient client;
private WebSocketClientImpl socketClient;
private Runnable reset;
@@ -128,18 +133,20 @@ public class ServerManager {
manager = new ProcessManager(GAV.INSTANCE.getGroupId(), mvnResolverImpl);
manager.start();
client = new WebSocketClient("ws://", String.valueOf(manager.getPort()), "/websocket/v1",
this.socketClient = new WebSocketClientImpl("ws://", String.valueOf(manager.getPort()), "/websocket/v1",
Long.getLong("talend.component.websocket.client.timeout", Constants.IO_TIMEOUT_MS_DEFAULT));
client.setSynch(() -> {
this.client = new ServicesClient(socketClient);
socketClient.setSynch(() -> {
manager.waitForServer(() -> {
client.setServerHost(manager.getServerAddress());
client.v1().healthCheck();
socketClient.setServerHost(manager.getServerAddress());
this.client.v1().healthCheck();
});
client.setServerHost(manager.getServerAddress());
socketClient.setServerHost(manager.getServerAddress());
});
services.add(ctx.registerService(ProcessManager.class.getName(), manager, new Hashtable<>()));
services.add(ctx.registerService(WebSocketClient.class.getName(), client, new Hashtable<>()));
services.add(ctx.registerService(ServicesClient.class.getName(), client, new Hashtable<>()));
services.add(ctx.registerService(ComponentService.class.getName(), new ComponentService(mvnResolverImpl),
new Hashtable<>()));
services.add(ctx.registerService(TaCoKitCache.class.getName(), new TaCoKitCache(), new Hashtable<>()));
@@ -156,6 +163,7 @@ public class ServerManager {
starting.set(false);
} catch (Throwable ex) {
CommonExceptionHandler.process(ex, Level.ERROR);
try {
doStop();
} catch (Exception e) {

View File

@@ -47,7 +47,7 @@ import org.talend.sdk.component.studio.lang.Pair;
import org.talend.sdk.component.studio.service.ComponentService;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.util.TaCokitImageUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
// note: for now we load the component on the server but
// we can use the mojo generating the meta later
@@ -66,7 +66,7 @@ public class TaCoKitGenericProvider implements IGenericProvider {
return;
}
final WebSocketClient client = Lookups.client();
final ServicesClient client = Lookups.client();
Stream<Pair<ComponentIndex, ComponentDetail>> details = client.v1().component().details(Locale.getDefault().getLanguage());
final ConfigTypeNodes configTypes = client.v1().configurationType().getRepositoryModel(true);

View File

@@ -0,0 +1,27 @@
package org.talend.sdk.component.studio.dependencies;
public class ComponentReference {
private final String family;
private final String name;
private final String mavenReferences;
public ComponentReference(String family, String name, String mavenReferences) {
this.family = family;
this.name = name;
this.mavenReferences = mavenReferences;
}
public String getFamily() {
return family;
}
public String getName() {
return name;
}
public String getMavenReferences() {
return mavenReferences;
}
}

View File

@@ -0,0 +1,80 @@
package org.talend.sdk.component.studio.dependencies;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.talend.core.model.process.IElementParameter;
import org.talend.core.model.process.INode;
import org.talend.sdk.component.server.front.model.ComponentDetail;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.model.parameter.ListPropertyNode;
import org.talend.sdk.component.studio.model.parameter.PropertyNode;
public interface ComponentReferenceFinder {
Stream<ComponentReference> find(final PropertyNode property, final INode node);
static ComponentReferenceFinder getFinder(final PropertyNode property) {
if (property instanceof ListPropertyNode) {
return new ListPropertyReferenceFinder();
}
return new PropertyReferenceFinder();
}
class PropertyReferenceFinder implements ComponentReferenceFinder {
@Override
public Stream<ComponentReference> find(PropertyNode property, INode node) {
final List<PropertyNode> children = property.getChildren();
String family = null;
String name = null;
String mavenReferences = null;
for (PropertyNode child : children) {
final String path = child.getProperty().getPath();
final IElementParameter elementParameter = node.getElementParameter(path);
final Object value = elementParameter.getValue();
if (value != null && path.endsWith(".family")) {
family = String.valueOf(value);
} else if (value != null && path.endsWith(".name")) {
name = String.valueOf(value);
} else if (value != null && path.endsWith(".mavenReferences")) {
mavenReferences = String.valueOf(value);
}
}
return Stream.of(new ComponentReference(family, name, mavenReferences));
}
}
class ListPropertyReferenceFinder implements ComponentReferenceFinder {
@Override
public Stream<ComponentReference> find(PropertyNode property, INode node) {
if (!(property instanceof ListPropertyNode)) {
return Stream.empty();
}
final List<ComponentReference> details = new ArrayList<>();
final String path = property.getProperty().getPath();
final IElementParameter elementParameter = node.getElementParameter(path);
final Object values = elementParameter.getValue();
if (values instanceof List) {
for (Object value : (List) values) {
if (value instanceof Map) {
final String family = this.extract((Map) value, "family");
final String name = this.extract((Map) value, "name");
final String mavenReferences = this.extract((Map) value, "mavenReferences");
details.add(new ComponentReference(family, name, mavenReferences));
}
}
}
return details.stream();
}
private String extract(Map values, String name) {
final Object value = ((Map) values).get('"' + name + '"');
if (!(value instanceof String)) {
return null;
}
return ((String) value).substring(1, ((String) value).length() - 1);
}
}
}

View File

@@ -0,0 +1,37 @@
package org.talend.sdk.component.studio.dependencies;
import java.util.Map;
import java.util.Optional;
import org.talend.sdk.component.studio.model.parameter.PropertyDefinitionDecorator;
import org.talend.sdk.component.studio.model.parameter.PropertyNode;
import org.talend.sdk.component.studio.model.parameter.TaCoKitElementParameter;
import org.talend.sdk.component.studio.model.parameter.TableElementParameter;
public class NodeFinder {
public PropertyNode find(final TaCoKitElementParameter parameter) {
final Optional<PropertyNode> propertyNode;
if (parameter instanceof TableElementParameter) {
propertyNode = Optional.ofNullable(parameter)
.map(TaCoKitElementParameter::getPropertyNode);
}
else {
propertyNode = Optional.ofNullable(parameter)
.map(TaCoKitElementParameter::getPropertyNode)
.map(PropertyNode::getParent);
}
Optional<Map<String, String>> metadata = propertyNode
.map(PropertyNode::getProperty)
.map(PropertyDefinitionDecorator::getMetadata);
if (metadata != null
&& metadata.isPresent()
&& metadata.get().containsKey("dependencies::connector")) {
return propertyNode.get();
}
return null;
}
}

View File

@@ -26,8 +26,8 @@ import org.talend.sdk.component.server.front.model.DocumentationContent;
import org.talend.sdk.component.server.front.model.ErrorDictionary;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.i18n.Messages;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.WebSocketClient.ClientException;
import org.talend.sdk.component.studio.websocket.ServicesClient;
import org.talend.sdk.component.studio.websocket.ServicesClient.ClientException;
public class TaCoKitHelpContentProducer implements IHelpContentProducer {
@@ -41,7 +41,7 @@ public class TaCoKitHelpContentProducer implements IHelpContentProducer {
@Override
public InputStream getInputStream(String pluginId, String href, Locale locale) {
String id = href;
final WebSocketClient client = Lookups.client();
final ServicesClient client = Lookups.client();
int index = id.lastIndexOf(".html");
if (index != -1) {
id = id.substring(0, index);

View File

@@ -36,14 +36,14 @@ import org.talend.sdk.component.studio.documentation.toc.TaCoKitTopic;
import org.talend.sdk.component.studio.lang.Pair;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
public class TaCoKitContextProvider extends AbstractContextProvider {
@Override
public IContext getContext(final String pluginId, final String contextName) {
final Locale expLocale = Locales.fromLanguagePresentation(contextName);
final WebSocketClient client = Lookups.client();
final ServicesClient client = Lookups.client();
//pluginId consists of two parts - plugin name and full component name and locale after the "."
//we will need to parse it to get the correct value of related topics.
final String fullComponentName = getFullComponentName(pluginId);

View File

@@ -34,7 +34,7 @@ import org.talend.sdk.component.studio.i18n.Messages;
import org.talend.sdk.component.studio.lang.Pair;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
public class TaCoKitTocProvider extends AbstractTocProvider {
@@ -58,7 +58,7 @@ public class TaCoKitTocProvider extends AbstractTocProvider {
if(contributions != null) {
return contributions;
}
final WebSocketClient client = Lookups.client();
final ServicesClient client = Lookups.client();
// we need to get the locale from display language. We might have a "en_US"/"en-US" or something different
// as an incoming locale String
final Locale expLocale = Locales.fromLanguagePresentation(language);

View File

@@ -30,11 +30,11 @@ import org.talend.sdk.component.server.front.model.ComponentDetail;
import org.talend.sdk.component.server.front.model.ComponentIndex;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.service.ComponentService;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
public class WizardRegistry {
private final WebSocketClient.V1Component client;
private final ServicesClient.V1Component client;
private final Constructor<ERepositoryObjectType> eRepositoryObjectTypeConstructor;

View File

@@ -59,7 +59,7 @@ import org.talend.sdk.component.studio.metadata.node.ITaCoKitRepositoryNode;
import org.talend.sdk.component.studio.model.parameter.PropertyDefinitionDecorator;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1Component;
import org.talend.sdk.component.studio.websocket.ServicesClient.V1Component;
import org.talend.sdk.studio.process.TaCoKitNode;
public class TaCoKitDragAndDropHandler extends AbstractDragAndDropServiceHandler {

View File

@@ -30,8 +30,8 @@ import org.talend.sdk.component.studio.exception.UserCancelledException;
import org.talend.sdk.component.studio.i18n.Messages;
import org.talend.sdk.component.studio.metadata.model.TaCoKitConfigurationModel;
import org.talend.sdk.component.studio.model.update.TaCoKitUpdateManager;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1Component;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1ConfigurationType;
import org.talend.sdk.component.studio.websocket.ServicesClient.V1Component;
import org.talend.sdk.component.studio.websocket.ServicesClient.V1ConfigurationType;
import org.talend.sdk.studio.process.TaCoKitNode;
/**

View File

@@ -37,7 +37,7 @@ import org.talend.core.model.process.IContextParameter;
import org.talend.designer.core.ui.editor.properties.controllers.uidialog.OpenContextChooseComboDialog;
import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.model.parameter.TableActionParameter;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1Action;
import org.talend.sdk.component.studio.websocket.ServicesClient.V1Action;
public class Action<T> {

View File

@@ -61,7 +61,7 @@ public final class ValueConverter {
record = trimCurlyBrackets(record);
String[] entries = record.split(",\\s?");Map<String, Object> element = new HashMap<String, Object>();
for (String entry : entries) {
String[] keyValue = entry.split("(:|=)");
String[] keyValue = entry.split("(:|=)", 2);
String key = keyValue[0];
String value = keyValue[1];
element.put(key, value);

View File

@@ -51,7 +51,7 @@ import org.talend.sdk.component.studio.Lookups;
import org.talend.sdk.component.studio.model.parameter.Metadatas;
import org.talend.sdk.component.studio.mvn.Mvn;
import org.talend.sdk.component.studio.util.TaCoKitUtil;
import org.talend.sdk.component.studio.websocket.WebSocketClient.V1Component;
import org.talend.sdk.component.studio.websocket.ServicesClient.V1Component;
public class ComponentService {

View File

@@ -0,0 +1,125 @@
package org.talend.sdk.component.studio.websocket;
import org.talend.sdk.component.server.front.model.error.ErrorPayload;
import javax.json.bind.Jsonb;
import javax.json.bind.spi.JsonbProvider;
import javax.json.stream.JsonParsingException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
class PayloadHandler implements Consumer<ByteBuffer> {
private static final byte[] EOM = "^@".getBytes(StandardCharsets.UTF_8);
private static final Jsonb jsonb = JsonbProvider.provider("org.apache.johnzon.jsonb.JohnzonProvider").create().build();
final CountDownLatch latch = new CountDownLatch(1);
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final byte[] lastBytes = new byte[2];
@Override
public void accept(final ByteBuffer byteBuffer) {
try {
final byte[] array = byteBuffer.array();
if (array.length >= 2) {
System.arraycopy(array, array.length - 2, lastBytes, 0, 2);
} else if (array.length > 0) {
lastBytes[0] = lastBytes[1];
lastBytes[1] = array[0];
}
out.write(array);
} catch (final IOException e) {
throw new IllegalStateException(e);
} finally {
if (Arrays.equals(lastBytes, EOM)) {
latch.countDown();
}
}
}
byte[] payload() {
return payload(true);
}
private byte[] payload(final boolean failOnBadStatus) {
final byte[] value = out.toByteArray();
// todo: check status header and fail if > 399 with the error message in the
// payload
int start = 0;
{
// find the first empty line which means the payload starts
boolean visitedEol = false;
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int idx = 0; idx < value.length - 1; idx++) {
if (value[idx] == '\r' && value[idx + 1] == '\n') {
if (failOnBadStatus) {
final String header = new String(baos.toByteArray(), StandardCharsets.UTF_8);
if (header.startsWith("status:")) {
try {
if (Integer.parseInt(header.substring("status:".length()).trim()) > 399) {
final String response = new String(value);
ErrorPayload errorPayload;
try {
errorPayload = PayloadHandler.parseResponse(payload(false), ErrorPayload.class);
} catch (final IllegalArgumentException iae) {
errorPayload = null;
}
throw new ServicesClient.ClientException(response, errorPayload);
}
} catch (final NumberFormatException nfe) {
}
}
}
// no-op: ignore this validation then
idx++;
if (visitedEol) {
start = idx + 1;
break;
}
visitedEol = true;
baos.reset();
} else {
baos.write(value[idx]);
visitedEol = false;
}
}
}
final int len = value.length - EOM.length - start;
if (len <= 0) {
return new byte[0];
}
final byte[] payload = new byte[len];
System.arraycopy(value, start, payload, 0, len);
return payload;
}
<T> T parseResponse(final Class<T> expectedResponse) {
return PayloadHandler.parseResponse(this.payload(), expectedResponse);
}
static <T> T parseResponse(final byte[] payload, final Class<T> expectedResponse) {
if (expectedResponse.isInstance(payload)) {
return expectedResponse.cast(payload);
}
if (String.class == expectedResponse) {
return expectedResponse.cast(new String(payload, StandardCharsets.UTF_8));
}
try (InputStream stream = new ByteArrayInputStream(payload)) {
return jsonb.fromJson(stream, expectedResponse);
} catch (final JsonParsingException jpe) {
throw new IllegalArgumentException("Can\'t parse JSON: \'" + new String(payload, StandardCharsets.UTF_8) + "\'", jpe);
} catch (final IOException e) {
throw new IllegalArgumentException(e);
}
}
}

View File

@@ -0,0 +1,217 @@
/**
* Copyright (C) 2006-2021 Talend Inc. - www.talend.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.talend.sdk.component.studio.websocket;
import org.talend.sdk.component.server.front.model.ActionList;
import org.talend.sdk.component.server.front.model.ComponentDetail;
import org.talend.sdk.component.server.front.model.ComponentDetailList;
import org.talend.sdk.component.server.front.model.ComponentIndex;
import org.talend.sdk.component.server.front.model.ComponentIndices;
import org.talend.sdk.component.server.front.model.ConfigTypeNodes;
import org.talend.sdk.component.server.front.model.DocumentationContent;
import org.talend.sdk.component.server.front.model.Environment;
import org.talend.sdk.component.server.front.model.error.ErrorPayload;
import org.talend.sdk.component.studio.lang.Pair;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toMap;
import javax.json.JsonObject;
// we shouldn't need the execution runtime so don't even include it here
//
// technical note: this client includes the transport (websocket) but also the protocol/payload formatting/parsing
// todo: better error handling, can need some server bridge love too to support ERROR responses
public class ServicesClient implements AutoCloseable {
private final WebSocketClient socketClient;
public ServicesClient(WebSocketClient socketClient) {
this.socketClient = socketClient;
}
public synchronized V1 v1() {
return new V1(this.socketClient);
}
@Override
public synchronized void close() {
this.socketClient.close();
}
public interface WebSocketClient extends AutoCloseable {
<T> T sendAndWait(final String id, final String uri, final Object payload, final Class<T> expectedResponse, final boolean doCheck);
void close();
}
public static class V1 {
private final WebSocketClient root;
V1(final WebSocketClient root) {
this.root = root;
}
public V1Action action() {
return new V1Action(root);
}
public V1ConfigurationType configurationType() {
return new V1ConfigurationType(root);
}
public V1Component component() {
return new V1Component(root);
}
public V1Documentation documentation() {
return new V1Documentation(root);
}
public boolean healthCheck() {
root.sendAndWait("/v1/get/environment", "/environment", null, Environment.class, false);
return true;
}
}
public static class V1ConfigurationType {
private final WebSocketClient root;
private V1ConfigurationType(final WebSocketClient root) {
this.root = root;
}
public ConfigTypeNodes getRepositoryModel() {
return root.sendAndWait("/v1/get/configurationtype/index", "/configurationtype/index?language=" + Locale.getDefault().getLanguage() + "&lightPayload=false", null, ConfigTypeNodes.class, true);
}
public ConfigTypeNodes getRepositoryModel(final boolean lightPayload) {
return root.sendAndWait("/v1/get/configurationtype/index", "/configurationtype/index?language=" + Locale.getDefault().getLanguage() + "&lightPayload="+lightPayload, null, ConfigTypeNodes.class, true);
}
public Map<String, String> migrate(final String id, final int configurationVersion, final Map<String, String> payload) {
return root.sendAndWait("/v1/post/configurationtype/migrate/{id}/{configurationVersion}",
"/configurationtype/migrate/" + id + "/" + configurationVersion, payload, Map.class, true);
}
}
public static class V1Action {
private final WebSocketClient root;
private V1Action(final WebSocketClient root) {
this.root = root;
}
public <T> T execute(final Class<T> expectedResponse, final String family, final String type, final String action, final Map<String, String> payload) {
return root.sendAndWait("/v1/post/action/execute", "/action/execute?family=" + family + "&type=" + type + "&action=" + action, payload, expectedResponse, true);
}
public ActionList getActionList(final String family) {
return root.sendAndWait("/v1/get/action/index/", "/action/index?family=" + family, null, ActionList.class, true);
}
}
public static class V1Documentation {
private final WebSocketClient root;
private V1Documentation(final WebSocketClient root) {
this.root = root;
}
public DocumentationContent getDocumentation(final String language, final String componentId, final String format) {
return root.sendAndWait("/v1/get/documentation/component",
"/documentation/component/" + componentId + "?language=" + language + "&format=" + format, null,
DocumentationContent.class, true);
}
}
public static class V1Component {
private static final int BUNDLE_SIZE = 25;
private final WebSocketClient root;
private V1Component(final WebSocketClient root) {
this.root = root;
}
public ComponentIndices getIndex(final String language) {
return root.sendAndWait("/v1/get/component/index", "/component/index?language=" + language + "&includeIconContent=true", null, ComponentIndices.class, true);
}
public Map<String, ?> dependencies(final String id) {
return root.sendAndWait("/v1/get/component/dependencies", "/component/dependencies?identifier=" + id, null, Map.class, true);
}
public byte[] icon(final String id) {
return root.sendAndWait("/v1/get/component/icon/" + id, "/component/icon/" + id, null, byte[].class, true);
}
public byte[] familyIcon(final String id) {
return root.sendAndWait("/v1/get/component/icon/family/" + id, "/component/icon/family/" + id, null, byte[].class, true);
}
public ComponentDetailList getDetail(final String language, final String[] identifiers) {
if (identifiers == null || identifiers.length == 0) {
return new ComponentDetailList(emptyList());
}
return root.sendAndWait("/v1/get/component/details", "/component/details?language=" + language + Stream.of(identifiers).map(i -> "identifiers=" + i).collect(Collectors.joining("&", "&", "")), null, ComponentDetailList.class, true);
}
public Stream<Pair<ComponentIndex, ComponentDetail>> details(final String language) {
final List<ComponentIndex> components = getIndex(language).getComponents();
// create bundles
int bundleCount = components.size() / BUNDLE_SIZE;
bundleCount = bundleCount * BUNDLE_SIZE >= components.size() ? bundleCount : (bundleCount + 1);
return IntStream.range(0, bundleCount).mapToObj(i -> {
final int from = BUNDLE_SIZE * i;
final int to = from + BUNDLE_SIZE;
return components.subList(from, Math.min(to, components.size()));
}).flatMap(bundle -> {
final Map<String, ComponentIndex> byId = bundle.stream().collect(toMap(c -> c.getId().getId(), identity()));
return getDetail(language, bundle.stream().map(i -> i.getId().getId()).toArray(String[]::new)).getDetails().stream().map(d -> new Pair<>(byId.get(d.getId().getId()), d));
});
}
public Map<String, String> migrate(final String id, final int configurationVersion, final Map<String, String> payload) {
return root.sendAndWait("/v1/post/component/migrate/{id}/{configurationVersion}", "/component/migrate/" + id + "/" + configurationVersion, payload, Map.class, true);
}
}
public static class ClientException extends RuntimeException {
private ErrorPayload errorPayload;
ClientException(final String raw, final ErrorPayload errorPayload) {
super(raw);
this.errorPayload = errorPayload;
}
public ErrorPayload getErrorPayload() {
return this.errorPayload;
}
}
}

View File

@@ -1,510 +0,0 @@
/**
* Copyright (C) 2006-2021 Talend Inc. - www.talend.com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.talend.sdk.component.studio.websocket;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.emptyList;
import static java.util.Locale.ENGLISH;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toMap;
import static org.talend.sdk.component.studio.util.TaCoKitConst.BASE64_PREFIX;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.json.bind.Jsonb;
import javax.json.bind.spi.JsonbProvider;
import javax.json.stream.JsonParsingException;
import javax.websocket.ClientEndpointConfig;
import javax.websocket.CloseReason;
import javax.websocket.ContainerProvider;
import javax.websocket.DeploymentException;
import javax.websocket.Endpoint;
import javax.websocket.EndpointConfig;
import javax.websocket.MessageHandler;
import javax.websocket.Session;
import javax.websocket.WebSocketContainer;
import org.apache.tomcat.websocket.Constants;
import org.talend.sdk.component.server.front.model.ActionList;
import org.talend.sdk.component.server.front.model.ComponentDetail;
import org.talend.sdk.component.server.front.model.ComponentDetailList;
import org.talend.sdk.component.server.front.model.ComponentIndex;
import org.talend.sdk.component.server.front.model.ComponentIndices;
import org.talend.sdk.component.server.front.model.ConfigTypeNodes;
import org.talend.sdk.component.server.front.model.DocumentationContent;
import org.talend.sdk.component.server.front.model.Environment;
import org.talend.sdk.component.server.front.model.error.ErrorPayload;
import org.talend.sdk.component.studio.lang.Pair;
import org.talend.sdk.component.studio.util.TaCoKitConst;
// we shouldn't need the execution runtime so don't even include it here
//
// technical note: this client includes the transport (websocket) but also the protocol/payload formatting/parsing
// todo: better error handling, can need some server bridge love too to support ERROR responses
public class WebSocketClient implements AutoCloseable {
private static final byte[] EOM = "^@".getBytes(UTF_8);
private final Queue<Session> sessions = new ConcurrentLinkedQueue<>();
private final WebSocketContainer container;
private final String protocol;
private final String port;
private final String basePath;
private final long timeout;
private final Jsonb jsonb;
private String serverHost;
private Runnable synch;
public WebSocketClient(final String protocol, final String port, final String basePath, final long timeout) {
this.protocol = protocol;
this.serverHost = TaCoKitConst.DEFAULT_LOCALHOST;
this.port = port;
this.basePath = basePath;
this.timeout = timeout;
this.container = ContainerProvider.getWebSocketContainer();
this.jsonb = JsonbProvider.provider("org.apache.johnzon.jsonb.JohnzonProvider").create().build();
}
public void setServerHost(String host) {
this.serverHost = host;
}
public void setSynch(final Runnable synch) {
this.synch = synch;
}
private String buildRequest(final String id, final String uri, final Object payload) {
final String method = id.substring("/v1/".length(), id.indexOf('/', "/v1/".length() + 1));
return "SEND\r\ndestination:" + uri + "\r\ndestinationMethod:" + method.toUpperCase(ENGLISH) + "\r\nAccept: application/json\r\nContent-Type: application/json\r\n\r\n" + (payload == null ? "" : jsonb.toJson(payload)) + "^@";
}
private <T> T parseResponse(final byte[] payload, final Class<T> expectedResponse) {
if (expectedResponse.isInstance(payload)) {
return expectedResponse.cast(payload);
}
if (String.class == expectedResponse) {
return expectedResponse.cast(new String(payload, UTF_8));
}
try (InputStream stream = new ByteArrayInputStream(payload)) {
return jsonb.fromJson(stream, expectedResponse);
} catch (final JsonParsingException jpe) {
throw new IllegalArgumentException("Can\'t parse JSON: \'" + new String(payload, UTF_8) + "\'");
} catch (final IOException e) {
throw new IllegalArgumentException(e);
}
}
private <T> T sendAndWait(final String id, final String uri, final Object payload, final Class<T> expectedResponse, final boolean doCheck) {
final Session session = getOrCreateSession(id, doCheck);
final PayloadHandler handler = new PayloadHandler(this);
session.getUserProperties().put("handler", handler);
final String buildRequest = buildRequest(id, uri, payload);
try {
try {
session.getBasicRemote().sendBinary(ByteBuffer.wrap(buildRequest.getBytes(StandardCharsets.UTF_8)));
} catch (final IOException e) {
throw new IllegalStateException(e);
}
try {
handler.latch.await(1, MINUTES); // todo: make it configurable? 1mn is already a lot
} catch (final InterruptedException e) {
Thread.interrupted();
throw new IllegalStateException(e);
}
} finally {
doRelease(session);
}
return parseResponse(handler.payload(), expectedResponse);
}
private void doRelease(final Session session) {
sessions.add(session);
}
private Session getOrCreateSession(final String id, final boolean doCheck) {
if (doCheck && synch != null) {
synchronized (this) {
if (synch != null) {
synch.run();
synch = null;
}
}
}
Session poll = sessions.poll();
if (poll != null && !poll.isOpen()) {
try {
poll.close(new CloseReason(CloseReason.CloseCodes.GOING_AWAY, "Session is no more opened"));
} catch (final Exception e) {
}
// just to go through close cycle but should fail since it is not opened, we
// just ignore any error
poll = null;
}
if (poll == null) {
poll = doConnect();
}
return poll;
}
private String getBase() {
return protocol + serverHost + ":" + port + basePath;
}
private Session doConnect() {
final URI connectUri = URI.create(getBase() + "/bus");
final ClientEndpointConfig endpointConfig = ClientEndpointConfig.Builder.create().build();
endpointConfig.getUserProperties().put(Constants.IO_TIMEOUT_MS_PROPERTY, Long.toString(timeout));
try {
return container.connectToServer(new Endpoint() {
@Override
public void onOpen(final Session session, final EndpointConfig endpointConfig) {
session.addMessageHandler(ByteBuffer.class, new MessageHandler.Partial<ByteBuffer>() {
@Override
public synchronized void onMessage(final ByteBuffer part, final boolean last) {
final Consumer<ByteBuffer> callback = Consumer.class.cast(session.getUserProperties()
.get("handler"));
callback.accept(part);
}
});
}
@Override
public void onError(final Session session, final Throwable throwable) {
final PayloadHandler handler = PayloadHandler.class.cast(session.getUserProperties()
.get("handler"));
if (handler != null) {
handler.latch.countDown();
}
throw new IllegalStateException(throwable);
}
}, endpointConfig, connectUri);
} catch (final DeploymentException | IOException e) {
throw new IllegalStateException(e);
}
}
@Override
public synchronized void close() {
sessions.forEach(s -> {
try {
s.close(new CloseReason(CloseReason.CloseCodes.GOING_AWAY, "Shutting down the studio"));
} catch (final IOException e) {
}
});
// no-op: todo: define if we want to log it, we will not do anything anyway at
// that time
sessions.clear();
}
public synchronized V1 v1() {
return new V1(this);
}
public static class V1 {
private final WebSocketClient root;
private V1(final WebSocketClient root) {
this.root = root;
}
public V1Action action() {
return new V1Action(root);
}
public V1ConfigurationType configurationType() {
return new V1ConfigurationType(root);
}
public V1Component component() {
return new V1Component(root);
}
public V1Documentation documentation() {
return new V1Documentation(root);
}
public boolean healthCheck() {
root.sendAndWait("/v1/get/environment", "/environment", null, Environment.class, false);
return true;
}
}
public static class V1ConfigurationType {
private final WebSocketClient root;
private V1ConfigurationType(final WebSocketClient root) {
this.root = root;
}
public ConfigTypeNodes getRepositoryModel() {
return root.sendAndWait("/v1/get/configurationtype/index", "/configurationtype/index?language=" + Locale.getDefault()
.getLanguage() + "&lightPayload=false", null, ConfigTypeNodes.class, true);
}
public ConfigTypeNodes getRepositoryModel(final boolean lightPayload) {
return root.sendAndWait("/v1/get/configurationtype/index", "/configurationtype/index?language=" + Locale.getDefault()
.getLanguage() + "&lightPayload=" + lightPayload, null, ConfigTypeNodes.class, true);
}
public Map<String, String> migrate(final String id, final int configurationVersion, final Map<String, String> payload) {
return root.sendAndWait("/v1/post/configurationtype/migrate/{id}/{configurationVersion}",
"/configurationtype/migrate/" + id + "/" + configurationVersion, payload, Map.class, true);
}
}
public static class V1Action {
private final WebSocketClient root;
private V1Action(final WebSocketClient root) {
this.root = root;
}
public <T> T execute(final Class<T> expectedResponse, final String family, final String type, final String action, final Map<String, String> payload) {
return root.sendAndWait("/v1/post/action/execute", "/action/execute?family=" + family + "&type=" + type + "&action=" + action, payload, expectedResponse, true);
}
public ActionList getActionList(final String family) {
return root.sendAndWait("/v1/get/action/index/", "/action/index?family=" + family, null, ActionList.class, true);
}
}
public static class V1Documentation {
private final WebSocketClient root;
private V1Documentation(final WebSocketClient root) {
this.root = root;
}
public DocumentationContent getDocumentation(final String language, final String componentId, final String format) {
return root.sendAndWait("/v1/get/documentation/component",
"/documentation/component/" + componentId + "?language=" + language + "&format=" + format, null,
DocumentationContent.class, true);
}
}
public static class V1Component {
private static final int BUNDLE_SIZE = 25;
private final WebSocketClient root;
private V1Component(final WebSocketClient root) {
this.root = root;
}
public ComponentIndices getIndex(final String language) {
return root.sendAndWait("/v1/get/component/index", "/component/index?language=" + language + "&includeIconContent=true", null, ComponentIndices.class, true);
}
public Map<String, ?> dependencies(final String id) {
return root.sendAndWait("/v1/get/component/dependencies", "/component/dependencies?identifier=" + id, null, Map.class, true);
}
public byte[] icon(final String id) {
return root.sendAndWait("/v1/get/component/icon/" + id, "/component/icon/" + id, null, byte[].class, true);
}
public byte[] familyIcon(final String id) {
return root.sendAndWait("/v1/get/component/icon/family/" + id, "/component/icon/family/" + id, null, byte[].class, true);
}
public ComponentDetailList getDetail(final String language, final String[] identifiers) {
if (identifiers == null || identifiers.length == 0) {
return new ComponentDetailList(emptyList());
}
return root.sendAndWait("/v1/get/component/details", "/component/details?language=" + language + Stream.of(identifiers)
.map(i -> "identifiers=" + i)
.collect(Collectors.joining("&", "&", "")), null, ComponentDetailList.class, true);
}
public Stream<Pair<ComponentIndex, ComponentDetail>> details(final String language) {
final List<ComponentIndex> components = getIndex(language).getComponents();
// create bundles
int bundleCount = components.size() / BUNDLE_SIZE;
bundleCount = bundleCount * BUNDLE_SIZE >= components.size() ? bundleCount : (bundleCount + 1);
return IntStream.range(0, bundleCount).mapToObj(i -> {
final int from = BUNDLE_SIZE * i;
final int to = from + BUNDLE_SIZE;
return components.subList(from, Math.min(to, components.size()));
}).flatMap(bundle -> {
final Map<String, ComponentIndex> byId = bundle.stream()
.collect(toMap(c -> c.getId().getId(), identity()));
return getDetail(language, bundle.stream().map(i -> i.getId().getId())
.toArray(String[]::new)).getDetails().stream()
.map(d -> new Pair<>(byId.get(d.getId().getId()), d));
});
}
public Map<String, String> migrate(final String id, final int configurationVersion, final Map<String, String> payload) {
Map<String, String> migrated = root.sendAndWait("/v1/post/component/migrate/{id}/{configurationVersion}", "/component/migrate/" + id + "/" + configurationVersion, payload, Map.class, true);
final Map<String, String> result = migrated.entrySet()
.stream()
.map(entry -> {
if (entry.getValue().startsWith(BASE64_PREFIX)) {
// it should have been encoded so decode
final String b64v = new String(Base64.getUrlDecoder().decode(entry.getValue().substring(BASE64_PREFIX.length()).getBytes(UTF_8)));
entry.setValue(b64v);
}
return entry;
})
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
return result;
}
}
private static class PayloadHandler implements Consumer<ByteBuffer> {
private final CountDownLatch latch = new CountDownLatch(1);
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
private final byte[] lastBytes = new byte[2];
private final WebSocketClient root;
public PayloadHandler(final WebSocketClient webSocketClient) {
root = webSocketClient;
}
@Override
public void accept(final ByteBuffer byteBuffer) {
try {
final byte[] array = byteBuffer.array();
if (array.length >= 2) {
System.arraycopy(array, array.length - 2, lastBytes, 0, 2);
} else if (array.length > 0) {
lastBytes[0] = lastBytes[1];
lastBytes[1] = array[0];
}
out.write(array);
} catch (final IOException e) {
throw new IllegalStateException(e);
} finally {
if (Arrays.equals(lastBytes, EOM)) {
latch.countDown();
}
}
}
private byte[] payload() {
return payload(true);
}
private byte[] payload(final boolean failOnBadStatus) {
final byte[] value = out.toByteArray();
// todo: check status header and fail if > 399 with the error message in the
// payload
int start = 0;
{
// find the first empty line which means the payload starts
boolean visitedEol = false;
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int idx = 0; idx < value.length - 1; idx++) {
if (value[idx] == '\r' && value[idx + 1] == '\n') {
if (failOnBadStatus) {
final String header = new String(baos.toByteArray(), UTF_8);
if (header.startsWith("status:")) {
try {
if (Integer.parseInt(header.substring("status:".length()).trim()) > 399) {
final String response = new String(value);
ErrorPayload errorPayload;
try {
errorPayload = root.parseResponse(payload(false), ErrorPayload.class);
} catch (final IllegalArgumentException iae) {
errorPayload = null;
}
throw new ClientException(response, errorPayload);
}
} catch (final NumberFormatException nfe) {
}
}
}
// no-op: ignore this validation then
idx++;
if (visitedEol) {
start = idx + 1;
break;
}
visitedEol = true;
baos.reset();
} else {
baos.write(value[idx]);
visitedEol = false;
}
}
}
final int len = value.length - EOM.length - start;
if (len <= 0) {
return new byte[0];
}
final byte[] payload = new byte[len];
System.arraycopy(value, start, payload, 0, len);
return payload;
}
}
public static class ClientException extends RuntimeException {
private ErrorPayload errorPayload;
private ClientException(final String raw, final ErrorPayload errorPayload) {
super(raw);
this.errorPayload = errorPayload;
}
public ErrorPayload getErrorPayload() {
return this.errorPayload;
}
}
}

View File

@@ -0,0 +1,168 @@
package org.talend.sdk.component.studio.websocket;
import static java.util.Locale.ENGLISH;
import static java.util.concurrent.TimeUnit.MINUTES;
import javax.json.bind.Jsonb;
import javax.json.bind.spi.JsonbProvider;
import javax.websocket.ClientEndpointConfig;
import javax.websocket.CloseReason;
import javax.websocket.ContainerProvider;
import javax.websocket.DeploymentException;
import javax.websocket.Endpoint;
import javax.websocket.EndpointConfig;
import javax.websocket.MessageHandler;
import javax.websocket.Session;
import javax.websocket.WebSocketContainer;
import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.function.Consumer;
import org.apache.tomcat.websocket.Constants;
public class WebSocketClientImpl implements ServicesClient.WebSocketClient {
private static final Jsonb jsonb = JsonbProvider.provider("org.apache.johnzon.jsonb.JohnzonProvider").create().build();
private final WebSocketContainer container;
private final String protocol;
private String serverHost;
private final String port;
private final String basePath;
private final long timeout;
private final Queue<Session> sessions = new ConcurrentLinkedQueue<>();
private Runnable synch;
public WebSocketClientImpl(String protocol, String port, String basePath, long timeout) {
this.container = ContainerProvider.getWebSocketContainer();
this.protocol = protocol;
this.port = port;
this.basePath = basePath;
this.timeout = timeout;
}
@Override
public <T> T sendAndWait(final String id, final String uri, final Object payload, final Class<T> expectedResponse, final boolean doCheck) {
final Session session = getOrCreateSession(id, doCheck);
final PayloadHandler handler = new PayloadHandler();
session.getUserProperties().put("handler", handler);
final String buildRequest = buildRequest(id, uri, payload);
try {
try {
session.getBasicRemote().sendBinary(ByteBuffer.wrap(buildRequest.getBytes(StandardCharsets.UTF_8)));
} catch (final IOException e) {
throw new IllegalStateException(e);
}
try {
handler.latch.await(1, MINUTES); // todo: make it configurable? 1mn is already a lot
} catch (final InterruptedException e) {
Thread.interrupted();
throw new IllegalStateException(e);
}
} finally {
doRelease(session);
}
return handler.parseResponse(expectedResponse);
}
private void doRelease(final Session session) {
sessions.add(session);
}
private String buildRequest(final String id, final String uri, final Object payload) {
final String method = id.substring("/v1/".length(), id.indexOf('/', "/v1/".length() + 1));
return "SEND\r\ndestination:" + uri + "\r\ndestinationMethod:" + method.toUpperCase(ENGLISH) + "\r\nAccept: application/json\r\nContent-Type: application/json\r\n\r\n" + (payload == null ? "" : jsonb.toJson(payload)) + "^@";
}
private Session getOrCreateSession(final String id, final boolean doCheck) {
if (doCheck && synch != null) {
synchronized (this) {
if (synch != null) {
synch.run();
synch = null;
}
}
}
Session poll = sessions.poll();
if (poll != null && !poll.isOpen()) {
try {
poll.close(new CloseReason(CloseReason.CloseCodes.GOING_AWAY, "Session is no more opened"));
} catch (final Exception e) {
}
// just to go through close cycle but should fail since it is not opened, we
// just ignore any error
poll = null;
}
if (poll == null) {
poll = doConnect();
}
return poll;
}
private Session doConnect() {
final URI connectUri = URI.create(getBase() + "/bus");
final ClientEndpointConfig endpointConfig = ClientEndpointConfig.Builder.create().build();
endpointConfig.getUserProperties().put(Constants.IO_TIMEOUT_MS_PROPERTY, Long.toString(timeout));
try {
return container.connectToServer(new Endpoint() {
@Override
public void onOpen(final Session session, final EndpointConfig endpointConfig) {
session.addMessageHandler(ByteBuffer.class, new MessageHandler.Partial<ByteBuffer>() {
@Override
public synchronized void onMessage(final ByteBuffer part, final boolean last) {
final Consumer<ByteBuffer> callback = Consumer.class.cast(session.getUserProperties().get("handler"));
callback.accept(part);
}
});
}
@Override
public void onError(final Session session, final Throwable throwable) {
final PayloadHandler handler = PayloadHandler.class.cast(session.getUserProperties().get("handler"));
if (handler != null) {
handler.latch.countDown();
}
throw new IllegalStateException(throwable);
}
}, endpointConfig, connectUri);
} catch (final DeploymentException | IOException e) {
throw new IllegalStateException(e);
}
}
private String getBase() {
return protocol + serverHost + ":" + port + basePath;
}
public void setServerHost(String serverHost) {
this.serverHost = serverHost;
}
public void setSynch(Runnable synch) {
this.synch = synch;
}
@Override
public synchronized void close() {
sessions.forEach(s -> {
try {
s.close(new CloseReason(CloseReason.CloseCodes.GOING_AWAY, "Shutting down the studio"));
} catch (final IOException e) {
}
});
// no-op: todo: define if we want to log it, we will not do anything anyway at
// that time
sessions.clear();
}
}

View File

@@ -67,7 +67,8 @@ import org.talend.sdk.component.server.front.model.ComponentIndices;
import org.talend.sdk.component.server.front.model.ConfigTypeNodes;
import org.talend.sdk.component.studio.mvn.Mvn;
import org.talend.sdk.component.studio.util.TaCoKitConst;
import org.talend.sdk.component.studio.websocket.WebSocketClient;
import org.talend.sdk.component.studio.websocket.ServicesClient;
import org.talend.sdk.component.studio.websocket.WebSocketClientImpl;
class ServerManagerTest {
@@ -147,7 +148,8 @@ class ServerManagerTest {
}
private void assertClient(final int port) {
try (WebSocketClient client = new WebSocketClient("ws://", String.valueOf(port), "/websocket/v1", 600000)) {
try (final ServicesClient.WebSocketClient socketClient = new WebSocketClientImpl("ws://", String.valueOf(port), "/websocket/v1", 600000);
final ServicesClient client = new ServicesClient(socketClient)) {
// we loop since we reuse the same session so we must ensure this reuse works
for (int i = 0; i < 2; i++) {
// simple endpoint
@@ -180,18 +182,18 @@ class ServerManagerTest {
}
}
private void assertIcons(final WebSocketClient client, final String compId, final String familyId) {
final WebSocketClient.V1Component component = client.v1().component();
private void assertIcons(final ServicesClient client, final String compId, final String familyId) {
final ServicesClient.V1Component component = client.v1().component();
try {
component.icon(compId);
} catch (final WebSocketClient.ClientException ce) {
} catch (final ServicesClient.ClientException ce) {
assertNotNull(ce.getErrorPayload());
assertEquals(ICON_MISSING, ce.getErrorPayload().getCode());
assertEquals("No icon for identifier: " + compId, ce.getErrorPayload().getDescription());
}
try {
component.familyIcon(familyId);
} catch (final WebSocketClient.ClientException ce) {
} catch (final ServicesClient.ClientException ce) {
assertNotNull(ce.getErrorPayload());
assertEquals(ICON_MISSING, ce.getErrorPayload().getCode());
assertEquals("No icon for family identifier: " + familyId, ce.getErrorPayload().getDescription());

View File

@@ -44,6 +44,19 @@ class ValueConverterTest {
Assertions.assertEquals(expected1, converted.get(1));
}
@Test
void toTable() {
final String table = "[{key1=value:11, key2=value12}, {key1=\"value::21\", key2=\"xx:value22\"}]";
final List<Map<String, Object>> converted = ValueConverter.toTable(table);
Assertions.assertEquals(2, converted.size());
Assertions.assertEquals("value:11", converted.get(0).get("key1"));
Assertions.assertEquals("value12", converted.get(0).get("key2"));
Assertions.assertEquals("\"value::21\"", converted.get(1).get("key1"));
Assertions.assertEquals("\"xx:value22\"", converted.get(1).get("key2"));
}
@Test
void testToTableNull() {
ArrayList<Map<String, Object>> empty = new ArrayList<>();

View File

@@ -0,0 +1,72 @@
package org.talend.sdk.component.studio.websocket;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.file.Files;
public class MockWebSocket implements ServicesClient.WebSocketClient {
private final MockReponseFile fileGetter;
@FunctionalInterface
interface MockReponseFile {
File getMockResponse(final String uri);
}
public static class DefaultFixFile implements MockReponseFile {
private final String fileName;
public DefaultFixFile(String fileName) {
this.fileName = fileName;
}
@Override
public File getMockResponse(String uri) {
final URL resource = Thread.currentThread().getContextClassLoader().getResource("websockettest");
return new File(resource.getPath(), this.fileName);
}
}
public MockWebSocket(final MockReponseFile fileGetter) {
this.fileGetter = fileGetter;
}
public MockWebSocket(final String fileName) {
this(new DefaultFixFile(fileName));
}
@Override
public <T> T sendAndWait(final String id,
final String uri,
final Object payload,
final Class<T> expectedResponse,
final boolean doCheck) {
final File mockSource = this.fileGetter.getMockResponse(uri);
return MockWebSocket.handle(mockSource, expectedResponse);
}
@Override
public void close() {
}
public static <T> T handle(final File mockSource,
final Class<T> expectedResponse) {
try {
final byte[] bytes = Files.readAllBytes(mockSource.toPath());
final byte[] allByte = new byte[bytes.length + 2];
System.arraycopy(bytes, 0, allByte, 0, bytes.length);
allByte[bytes.length] = '^';
allByte[bytes.length + 1] = '@';
final PayloadHandler handler = new PayloadHandler();
final ByteBuffer byteBuffer = ByteBuffer.wrap(allByte);
handler.accept(byteBuffer);
return handler.parseResponse(expectedResponse);
} catch (IOException ex) {
throw new UncheckedIOException("unreadable file " + mockSource.getPath(), ex);
}
}
}

View File

@@ -0,0 +1,38 @@
package org.talend.sdk.component.studio.websocket;
import java.util.Collection;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.talend.sdk.component.server.front.model.ConfigTypeNode;
import org.talend.sdk.component.server.front.model.ConfigTypeNodes;
class ServicesClientTest {
@Test
void v1ComponentDependencies() {
final ServicesClient.WebSocketClient ws = new MockWebSocket("dependencies.json");
final ServicesClient client = new ServicesClient(ws);
final Map<String, ?> dependencies1 = client.v1().component().dependencies("123");
final Collection<String> coordinates = Collection.class.cast(Map.class
.cast(Map.class.cast(dependencies1.get("dependencies")).values().iterator().next())
.get("dependencies"));
Assertions.assertNotNull(coordinates);
Assertions.assertTrue(coordinates.contains("org.talend.components:common:jar:1.30.0"));
}
@Test
void v1getRepositoryModel() {
final ServicesClient.WebSocketClient ws = new MockWebSocket("configType.json");
final ServicesClient client = new ServicesClient(ws);
final ConfigTypeNodes configTypeNodes = client.v1().configurationType().getRepositoryModel();
Assertions.assertNotNull(configTypeNodes);
final ConfigTypeNode datasetNode = configTypeNodes.getNodes().get("cmVzdCNSRVNUI2RhdGFzZXQjRGF0YXNldA");
Assertions.assertEquals("Dataset", datasetNode.getName());
}
}

View File

@@ -0,0 +1,936 @@
{
"nodes": {
"cmVzdCNSRVNUI2RhdGFzZXQjRGF0YXNldA": {
"actions": [],
"configurationType": "dataset",
"displayName": "Batch",
"edges": [],
"id": "cmVzdCNSRVNUI2RhdGFzZXQjRGF0YXNldA",
"name": "Dataset",
"parentId": "cmVzdCNSRVNUI2RhdGFzdG9yZSNEYXRhc3RvcmU",
"properties": [
{
"displayName": "",
"metadata": {
"ui::gridlayout::Advanced::value": "datastore|completePayload|maxRedirect|only_same_host|force_302_redirect|jsonForceDouble",
"ui::gridlayout::Main::value": "datastore|resource|methodType|format|hasHeaders|headers|hasQueryParams|queryParams|hasPathParams|pathParams|hasBody|body",
"documentation::value": "Dataset configuration.",
"configurationtype::name": "Dataset",
"configurationtype::type": "dataset",
"definition::parameter::index": "0"
},
"name": "configuration",
"path": "configuration",
"placeholder": "configuration",
"type": "OBJECT"
},
{
"displayName": "Body",
"metadata": {
"ui::gridlayout::Main::value": "type|textValue|jsonValue|xmlValue|params",
"condition::if::target": "hasBody",
"documentation::value": "Request body.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "true",
"condition::if::negate": "false"
},
"name": "body",
"path": "configuration.body",
"placeholder": "body",
"type": "OBJECT"
},
{
"displayName": "JSON",
"metadata": {
"condition::if::target": "type",
"ui::code::value": "json",
"documentation::value": "JSON content of the body.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "JSON",
"condition::if::negate": "false"
},
"name": "jsonValue",
"path": "configuration.body.jsonValue",
"placeholder": "{}",
"type": "STRING"
},
{
"defaultValue": "[]",
"displayName": "Body parameters",
"metadata": {
"ui::gridlayout::Main::value": "key,value",
"condition::if::target": "type",
"documentation::value": "Body parameters.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "FORM_DATA,X_WWW_FORM_URLENCODED",
"condition::if::negate": "false"
},
"name": "params",
"path": "configuration.body.params",
"placeholder": "params",
"type": "ARRAY"
},
{
"displayName": "Name",
"metadata": {
"documentation::value": "Name of the parameter."
},
"name": "key",
"path": "configuration.body.params[].key",
"placeholder": "key",
"type": "STRING",
"validation": {
"required": true
}
},
{
"displayName": "Value",
"metadata": {
"documentation::value": "Value of the parameter."
},
"name": "value",
"path": "configuration.body.params[].value",
"placeholder": "value",
"type": "STRING"
},
{
"displayName": "Text",
"metadata": {
"ui::textarea": "true",
"condition::if::target": "type",
"documentation::value": "TExt content of the body.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "TEXT",
"condition::if::negate": "false"
},
"name": "textValue",
"path": "configuration.body.textValue",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "TEXT",
"displayName": "Body type",
"metadata": {
"documentation::value": "Type of the request body."
},
"name": "type",
"path": "configuration.body.type",
"placeholder": "type",
"proposalDisplayNames": {
"TEXT": "Text",
"JSON": "JSON",
"XML": "XML",
"FORM_DATA": "Form data",
"X_WWW_FORM_URLENCODED": "x-www-form-urlencoded"
},
"type": "ENUM",
"validation": {
"enumValues": [
"TEXT",
"JSON",
"XML",
"FORM_DATA",
"X_WWW_FORM_URLENCODED"
]
}
},
{
"displayName": "XML",
"metadata": {
"condition::if::target": "type",
"ui::code::value": "xml",
"documentation::value": "XML content of the body.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "XML",
"condition::if::negate": "false"
},
"name": "xmlValue",
"path": "configuration.body.xmlValue",
"placeholder": "<xml></xml>",
"type": "STRING"
},
{
"defaultValue": "false",
"displayName": "Get status and headers",
"metadata": {
"documentation::value": "Return complete payload as a record."
},
"name": "completePayload",
"path": "configuration.completePayload",
"placeholder": "completePayload",
"type": "BOOLEAN"
},
{
"displayName": "",
"metadata": {
"ui::gridlayout::Advanced::value": "connectionTimeout|readTimeout",
"ui::gridlayout::Main::value": "base|authentication|dep|deps",
"documentation::value": "Identification of the REST API.",
"configurationtype::name": "Datastore",
"configurationtype::type": "datastore"
},
"name": "datastore",
"path": "configuration.datastore",
"placeholder": "datastore",
"type": "OBJECT"
},
{
"displayName": "Authentication",
"metadata": {
"ui::gridlayout::Main::value": "type|basic|bearerToken",
"documentation::value": "Authentication configuration."
},
"name": "authentication",
"path": "configuration.datastore.authentication",
"placeholder": "authentication",
"type": "OBJECT",
"validation": {
"required": true
}
},
{
"displayName": "",
"metadata": {
"ui::gridlayout::Main::value": "username,password",
"condition::if::target": "type",
"documentation::value": "Login/password authentication.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "Basic,Digest",
"condition::if::negate": "false"
},
"name": "basic",
"path": "configuration.datastore.authentication.basic",
"placeholder": "basic",
"type": "OBJECT"
},
{
"displayName": "Password",
"metadata": {
"documentation::value": "Password for authentication.",
"ui::credential": "true"
},
"name": "password",
"path": "configuration.datastore.authentication.basic.password",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "Username",
"metadata": {
"documentation::value": "Username for authentication."
},
"name": "username",
"path": "configuration.datastore.authentication.basic.username",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "Bearer access token",
"metadata": {
"condition::if::target": "type",
"documentation::value": "Bearer token.",
"ui::credential": "true",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "Bearer",
"condition::if::negate": "false"
},
"name": "bearerToken",
"path": "configuration.datastore.authentication.bearerToken",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "NoAuth",
"displayName": "Authentication type",
"metadata": {
"documentation::value": "Request authentication type."
},
"name": "type",
"path": "configuration.datastore.authentication.type",
"placeholder": "Authentication type",
"proposalDisplayNames": {
"NoAuth": "No authentication",
"Basic": "Basic",
"Digest": "Digest",
"Bearer": "Bearer token"
},
"type": "ENUM",
"validation": {
"enumValues": [
"NoAuth",
"Basic",
"Digest",
"Bearer"
]
}
},
{
"defaultValue": "",
"displayName": "Base URL",
"metadata": {
"documentation::value": "URL base of the request."
},
"name": "base",
"path": "configuration.datastore.base",
"placeholder": "https://www.restapi.org",
"type": "STRING",
"validation": {
"pattern": "^https?://.+$",
"required": true
}
},
{
"defaultValue": "30000",
"displayName": "Connection timeout (ms)",
"metadata": {
"ui::defaultvalue::value": "30000",
"documentation::value": "Connection timeout (ms)."
},
"name": "connectionTimeout",
"path": "configuration.datastore.connectionTimeout",
"placeholder": "30000",
"type": "NUMBER",
"validation": {
"min": 0,
"required": true
}
},
{
"displayName": "dep",
"metadata": {
"dependencies::connector": "family",
"documentation::value": "One deps."
},
"name": "dep",
"path": "configuration.datastore.dep",
"placeholder": "",
"type": "OBJECT"
},
{
"displayName": "family",
"metadata": {},
"name": "family",
"path": "configuration.datastore.dep.family",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "mavenReferences",
"metadata": {},
"name": "mavenReferences",
"path": "configuration.datastore.dep.mavenReferences",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "name",
"metadata": {},
"name": "name",
"path": "configuration.datastore.dep.name",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "[]",
"displayName": "deps",
"metadata": {
"dependencies::connector": "family",
"documentation::value": "Several deps."
},
"name": "deps",
"path": "configuration.datastore.deps",
"placeholder": "",
"type": "ARRAY"
},
{
"displayName": "family",
"metadata": {},
"name": "family",
"path": "configuration.datastore.deps[].family",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "mavenReferences",
"metadata": {},
"name": "mavenReferences",
"path": "configuration.datastore.deps[].mavenReferences",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "name",
"metadata": {},
"name": "name",
"path": "configuration.datastore.deps[].name",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "120000",
"displayName": "Read timeout (ms)",
"metadata": {
"ui::defaultvalue::value": "120000",
"documentation::value": "Read timeout (ms)."
},
"name": "readTimeout",
"path": "configuration.datastore.readTimeout",
"placeholder": "120000",
"type": "NUMBER",
"validation": {
"min": 0,
"required": true
}
},
{
"defaultValue": "false",
"displayName": "Force GET method for 302 redirection",
"metadata": {
"ui::defaultvalue::value": "false",
"condition::if::target": "maxRedirect",
"documentation::value": "Force a GET on a 302 redirection.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "0",
"condition::if::negate": "true"
},
"name": "force_302_redirect",
"path": "configuration.force_302_redirect",
"placeholder": "force_302_redirect",
"type": "BOOLEAN"
},
{
"defaultValue": "RAW_TEXT",
"displayName": "Answer body format",
"metadata": {
"documentation::value": "Format of the body's answer."
},
"name": "format",
"path": "configuration.format",
"placeholder": "",
"proposalDisplayNames": {
"RAW_TEXT": "Text",
"JSON": "JSON"
},
"type": "ENUM",
"validation": {
"enumValues": [
"RAW_TEXT",
"JSON"
],
"required": true
}
},
{
"defaultValue": "false",
"displayName": "Define body",
"metadata": {
"documentation::value": "Activate to define the body."
},
"name": "hasBody",
"path": "configuration.hasBody",
"placeholder": "hasBody",
"type": "BOOLEAN"
},
{
"defaultValue": "false",
"displayName": "Set header",
"metadata": {
"documentation::value": "Activate to define headers."
},
"name": "hasHeaders",
"path": "configuration.hasHeaders",
"placeholder": "hasHeaders",
"type": "BOOLEAN"
},
{
"defaultValue": "false",
"displayName": "Set path parameters",
"metadata": {
"documentation::value": "Activate to define URL path parameters."
},
"name": "hasPathParams",
"path": "configuration.hasPathParams",
"placeholder": "hasPathParams",
"type": "BOOLEAN"
},
{
"defaultValue": "false",
"displayName": "Set URL parameters",
"metadata": {
"documentation::value": "Activate to define query parameters."
},
"name": "hasQueryParams",
"path": "configuration.hasQueryParams",
"placeholder": "hasQueryParams",
"type": "BOOLEAN"
},
{
"defaultValue": "[{\"key\":\"\",\"value\":\"\"}]",
"displayName": "Headers",
"metadata": {
"ui::gridlayout::Main::value": "key,value",
"condition::if::target": "hasHeaders",
"documentation::value": "Query headers.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "true",
"condition::if::negate": "false"
},
"name": "headers",
"path": "configuration.headers",
"placeholder": "headers",
"type": "ARRAY"
},
{
"defaultValue": "",
"displayName": "Name",
"metadata": {
"documentation::value": "Name of the parameter."
},
"name": "key",
"path": "configuration.headers[].key",
"placeholder": "key",
"type": "STRING",
"validation": {
"required": true
}
},
{
"defaultValue": "",
"displayName": "Value",
"metadata": {
"documentation::value": "Value of the parameter."
},
"name": "value",
"path": "configuration.headers[].value",
"placeholder": "value",
"type": "STRING"
},
{
"defaultValue": "true",
"displayName": "Force all JSON numbers to double when parsing the answer body.",
"metadata": {
"ui::defaultvalue::value": "true",
"condition::if::target": "format",
"documentation::value": "If answer body type is JSON, infer numbers type or force all to double.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "JSON",
"condition::if::negate": "false"
},
"name": "jsonForceDouble",
"path": "configuration.jsonForceDouble",
"placeholder": "",
"type": "BOOLEAN"
},
{
"defaultValue": "3",
"displayName": "Maximum number of redirects",
"metadata": {
"ui::defaultvalue::value": "3",
"documentation::value": "How many redirection are supported ? (-1 for infinite)."
},
"name": "maxRedirect",
"path": "configuration.maxRedirect",
"placeholder": "3",
"type": "NUMBER",
"validation": {
"min": -1,
"required": true
}
},
{
"displayName": "HTTP method",
"metadata": {
"ui::defaultvalue::value": "GET",
"documentation::value": "The HTTP verb to use."
},
"name": "methodType",
"path": "configuration.methodType",
"placeholder": "",
"proposalDisplayNames": {
"GET": "GET",
"POST": "POST",
"HEAD": "HEAD",
"OPTIONS": "Options",
"PUT": "PUT",
"DELETE": "DELETE",
"TRACE": "Trace"
},
"type": "ENUM",
"validation": {
"enumValues": [
"GET",
"POST",
"HEAD",
"OPTIONS",
"PUT",
"DELETE",
"TRACE"
],
"required": true
}
},
{
"defaultValue": "false",
"displayName": "Redirect only on same host",
"metadata": {
"ui::defaultvalue::value": "false",
"condition::if::target": "maxRedirect",
"documentation::value": "Redirect only if same host.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "0",
"condition::if::negate": "true"
},
"name": "only_same_host",
"path": "configuration.only_same_host",
"placeholder": "only_same_host",
"type": "BOOLEAN"
},
{
"defaultValue": "[{\"key\":\"\",\"value\":\"\"}]",
"displayName": "Parameters",
"metadata": {
"ui::gridlayout::Main::value": "key,value",
"condition::if::target": "hasPathParams",
"documentation::value": "Path parameters.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "true",
"condition::if::negate": "false"
},
"name": "pathParams",
"path": "configuration.pathParams",
"placeholder": "pathParams",
"type": "ARRAY"
},
{
"defaultValue": "",
"displayName": "Name",
"metadata": {
"documentation::value": "Name of the parameter."
},
"name": "key",
"path": "configuration.pathParams[].key",
"placeholder": "key",
"type": "STRING",
"validation": {
"required": true
}
},
{
"defaultValue": "",
"displayName": "Value",
"metadata": {
"documentation::value": "Value of the parameter."
},
"name": "value",
"path": "configuration.pathParams[].value",
"placeholder": "value",
"type": "STRING"
},
{
"defaultValue": "[{\"key\":\"\",\"value\":\"\"}]",
"displayName": "Query parameters",
"metadata": {
"ui::gridlayout::Main::value": "key,value",
"condition::if::target": "hasQueryParams",
"documentation::value": "Query parameters.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "true",
"condition::if::negate": "false"
},
"name": "queryParams",
"path": "configuration.queryParams",
"placeholder": "queryParams",
"type": "ARRAY"
},
{
"defaultValue": "",
"displayName": "Name",
"metadata": {
"documentation::value": "Name of the parameter."
},
"name": "key",
"path": "configuration.queryParams[].key",
"placeholder": "key",
"type": "STRING",
"validation": {
"required": true
}
},
{
"defaultValue": "",
"displayName": "Value",
"metadata": {
"documentation::value": "Value of the parameter."
},
"name": "value",
"path": "configuration.queryParams[].value",
"placeholder": "value",
"type": "STRING"
},
{
"defaultValue": "",
"displayName": "Resource",
"metadata": {
"documentation::value": "End of url to complete base url of the datastore."
},
"name": "resource",
"path": "configuration.resource",
"placeholder": "/get",
"type": "STRING",
"validation": {
"required": true
}
}
],
"version": -1
},
"cmVzdCNSRVNU": {
"displayName": "REST",
"edges": [
"cmVzdCNSRVNUI2RhdGFzdG9yZSNEYXRhc3RvcmU"
],
"id": "cmVzdCNSRVNU",
"name": "REST",
"properties": [],
"version": 0
},
"cmVzdCNSRVNUI2RhdGFzdG9yZSNEYXRhc3RvcmU": {
"actions": [],
"configurationType": "datastore",
"displayName": "REST",
"edges": [
"cmVzdCNSRVNUI2RhdGFzZXQjRGF0YXNldA"
],
"id": "cmVzdCNSRVNUI2RhdGFzdG9yZSNEYXRhc3RvcmU",
"name": "Datastore",
"parentId": "cmVzdCNSRVNU",
"properties": [
{
"displayName": "",
"metadata": {
"ui::gridlayout::Advanced::value": "connectionTimeout|readTimeout",
"ui::gridlayout::Main::value": "base|authentication|dep|deps",
"documentation::value": "Identification of the REST API.",
"configurationtype::name": "Datastore",
"configurationtype::type": "datastore",
"definition::parameter::index": "0"
},
"name": "configuration",
"path": "configuration",
"placeholder": "configuration",
"type": "OBJECT"
},
{
"displayName": "Authentication",
"metadata": {
"ui::gridlayout::Main::value": "type|basic|bearerToken",
"documentation::value": "Authentication configuration."
},
"name": "authentication",
"path": "configuration.authentication",
"placeholder": "authentication",
"type": "OBJECT",
"validation": {
"required": true
}
},
{
"displayName": "",
"metadata": {
"ui::gridlayout::Main::value": "username,password",
"condition::if::target": "type",
"documentation::value": "Login/password authentication.",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "Basic,Digest",
"condition::if::negate": "false"
},
"name": "basic",
"path": "configuration.authentication.basic",
"placeholder": "basic",
"type": "OBJECT"
},
{
"displayName": "Password",
"metadata": {
"documentation::value": "Password for authentication.",
"ui::credential": "true"
},
"name": "password",
"path": "configuration.authentication.basic.password",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "Username",
"metadata": {
"documentation::value": "Username for authentication."
},
"name": "username",
"path": "configuration.authentication.basic.username",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "Bearer access token",
"metadata": {
"condition::if::target": "type",
"documentation::value": "Bearer token.",
"ui::credential": "true",
"condition::if::evaluationStrategy": "DEFAULT",
"condition::if::value": "Bearer",
"condition::if::negate": "false"
},
"name": "bearerToken",
"path": "configuration.authentication.bearerToken",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "NoAuth",
"displayName": "Authentication type",
"metadata": {
"documentation::value": "Request authentication type."
},
"name": "type",
"path": "configuration.authentication.type",
"placeholder": "Authentication type",
"proposalDisplayNames": {
"NoAuth": "No authentication",
"Basic": "Basic",
"Digest": "Digest",
"Bearer": "Bearer token"
},
"type": "ENUM",
"validation": {
"enumValues": [
"NoAuth",
"Basic",
"Digest",
"Bearer"
]
}
},
{
"defaultValue": "",
"displayName": "Base URL",
"metadata": {
"documentation::value": "URL base of the request."
},
"name": "base",
"path": "configuration.base",
"placeholder": "https://www.restapi.org",
"type": "STRING",
"validation": {
"pattern": "^https?://.+$",
"required": true
}
},
{
"defaultValue": "30000",
"displayName": "Connection timeout (ms)",
"metadata": {
"ui::defaultvalue::value": "30000",
"documentation::value": "Connection timeout (ms)."
},
"name": "connectionTimeout",
"path": "configuration.connectionTimeout",
"placeholder": "30000",
"type": "NUMBER",
"validation": {
"min": 0,
"required": true
}
},
{
"displayName": "dep",
"metadata": {
"dependencies::connector": "family",
"documentation::value": "One deps."
},
"name": "dep",
"path": "configuration.dep",
"placeholder": "",
"type": "OBJECT"
},
{
"displayName": "family",
"metadata": {},
"name": "family",
"path": "configuration.dep.family",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "mavenReferences",
"metadata": {},
"name": "mavenReferences",
"path": "configuration.dep.mavenReferences",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "name",
"metadata": {},
"name": "name",
"path": "configuration.dep.name",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "[]",
"displayName": "deps",
"metadata": {
"dependencies::connector": "family",
"documentation::value": "Several deps."
},
"name": "deps",
"path": "configuration.deps",
"placeholder": "",
"type": "ARRAY"
},
{
"displayName": "family",
"metadata": {},
"name": "family",
"path": "configuration.deps[].family",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "mavenReferences",
"metadata": {},
"name": "mavenReferences",
"path": "configuration.deps[].mavenReferences",
"placeholder": "",
"type": "STRING"
},
{
"displayName": "name",
"metadata": {},
"name": "name",
"path": "configuration.deps[].name",
"placeholder": "",
"type": "STRING"
},
{
"defaultValue": "120000",
"displayName": "Read timeout (ms)",
"metadata": {
"ui::defaultvalue::value": "120000",
"documentation::value": "Read timeout (ms)."
},
"name": "readTimeout",
"path": "configuration.readTimeout",
"placeholder": "120000",
"type": "NUMBER",
"validation": {
"min": 0,
"required": true
}
}
],
"version": 1
}
}
}

View File

@@ -0,0 +1,29 @@
{
"dependencies": {
"cmVzdCNSRVNUI0lucHV0": {
"dependencies": [
"org.talend.components:rest-locales:jar:1.28.1",
"org.talend.components:common-io:jar:1.30.0",
"org.talend.components:common-io-locales:jar:1.30.0",
"org.talend.components:stream-json:jar:1.30.0",
"org.talend.components:stream-api:jar:1.30.0",
"org.talend.components:stream-api-locales:jar:1.30.0",
"org.talend.components:stream-rawtext:jar:1.30.0",
"org.talend.components:common:jar:1.30.0",
"org.talend.components.extension:polling:jar:1.30.0",
"org.talend.components.extension:polling-locales:jar:1.30.0",
"org.talend.sdk.component:component-runtime-design-extension:jar:1.41.0",
"org.talend.sdk.component:component-runtime-manager:jar:1.41.0",
"org.talend.sdk.component:component-spi:jar:1.41.0",
"org.talend.sdk.component:component-runtime-impl:jar:1.41.0",
"org.apache.johnzon:johnzon-jsonb:jar:1.2.15",
"org.apache.johnzon:johnzon-mapper:jar:1.2.15",
"org.talend.sdk.component:container-core:jar:1.41.0",
"org.apache.xbean:xbean-finder-shaded:jar:4.20",
"org.apache.xbean:xbean-reflect:jar:4.20",
"org.apache.xbean:xbean-asm9-shaded:jar:4.20",
"org.apache.johnzon:johnzon-core:jar:1.2.15"
]
}
}
}