Compare commits

...

13 Commits

Author SHA1 Message Date
Chao MENG
2ae1b236b4 chore: fix customURIObject null issue 2020-11-20 10:04:22 +08:00
apoltavtsev
11283d417f feat(TESB-29553) Correct build issue due to duplicated methods 2020-11-17 12:03:29 +01:00
bhe-talendbj
bdc8061ea8 feat(TESB-28330): remove workaround for cConfig (#3755)
* feat(TESB-28330): remove workaround for cConfig

* TESB-28330:cConfig/Beans - Support customer groupid and artifact name
(additional to custom version)

* TESB-28330:cConfig/Beans - Support customer groupid and artifact name
(additional to custom version)

Co-authored-by: SunChaoqun <csun@talend.com>
2020-11-17 13:41:51 +08:00
kjwang
e9abdf6fee Fix TUP-29290 "Detect and Update all jobs " is not identifying the changes in imported projects. (#3833)
Fix TUP-29290 "Detect and Update all jobs " is not identifying the
changes in imported projects.
https://jira.talendforge.org/browse/TUP-29290
2020-11-17 11:19:38 +08:00
zyuan-talend
87fe04566a TUP-28024: add automatic Nexus3 testing (#3844) 2020-11-17 10:04:20 +08:00
apoltavtsev
2e30f29ca2 fix(TESB-29553) Publishing a route with cTalendJob from Studio and commandline gives different results (#3846)
* fix(TESB-29553) Publishing a route with cTalendJob from Studio and commandline gives different results

* fix(TESB-29553) Publishing a route with cTalendJob from Studio and commandline gives different results

* fix(TESB-29553) Publishing a route with cTalendJob from Studio and commandline gives different results
2020-11-16 15:00:48 +01:00
Chao MENG
fb2130e478 fix(TUP-29285): Improve reloading logic of Custom Uri Mapping (#3829)
https://jira.talendforge.org/browse/TUP-29285
2020-11-12 17:12:56 +08:00
jiezhang-tlnd
90a880096b fix(28893)fix junits (#3832) 2020-11-12 11:18:57 +08:00
Zhiwei Xue
65e8b699eb fix(TUP-29265):"<routine_name> cannot be resolved" error after (#3826)
installing latest R2020-10 patch
2020-11-12 09:55:12 +08:00
hzhao-talendbj
f6f8377627 change version (#3828) 2020-11-11 16:36:51 +08:00
jiezhang-tlnd
b0c52135cd fix(TUP-28893)Support more types of artifact for share libs (#3804)
* fix(TUP-28893)Support more types of artifact for share libs
https://jira.talendforge.org/browse/TUP-28893

* fix(TUP-28893)Support more types of artifact for share libs
2020-11-11 16:15:21 +08:00
Chao MENG
4af0090cce feat(TUP-21457): License fetch feature with the https proxy (#3695)
feat(TUP-21457): License fetch feature with the https proxy
https://jira.talendforge.org/browse/TUP-21457
2020-11-11 14:21:04 +08:00
chmyga
461dff94d8 chore(TDI-45157): bump tck components version (#3822)
Co-authored-by: Dmytro Chmyga <dmytro.chmyga@synapse.com>
2020-11-10 14:41:06 +02:00
30 changed files with 1039 additions and 180 deletions

View File

@@ -58,6 +58,14 @@ public class NetworkUtil {
public static final String ORG_TALEND_DESIGNER_CORE = "org.talend.designer.core"; //$NON-NLS-1$
private static final String PROP_DISABLEDSCHEMES_USE_DEFAULT = "talend.studio.jdk.http.auth.tunneling.disabledSchemes.useDefault";
private static final String PROP_JRE_DISABLEDSCHEMES = "jdk.http.auth.tunneling.disabledSchemes";
private static final String PROP_JRE_DISABLEDSCHEMES_DFAULT = "";
private static final String PROP_HTTP_PROXY_SET = "http.proxySet";
public static boolean isNetworkValid() {
return isNetworkValid(DEFAULT_TIMEOUT);
}
@@ -205,6 +213,21 @@ public class NetworkUtil {
} else {
Authenticator.setDefault(null);
}
checkProxyAuthSupport();
}
public static void checkProxyAuthSupport() {
if (!Boolean.getBoolean(PROP_DISABLEDSCHEMES_USE_DEFAULT)) {
if (Boolean.getBoolean(PROP_HTTP_PROXY_SET)) {
if (!System.getProperties().containsKey(PROP_JRE_DISABLEDSCHEMES)) {
System.setProperty(PROP_JRE_DISABLEDSCHEMES, PROP_JRE_DISABLEDSCHEMES_DFAULT);
}
} else {
if (PROP_JRE_DISABLEDSCHEMES_DFAULT.equals(System.getProperty(PROP_JRE_DISABLEDSCHEMES))) {
System.getProperties().remove(PROP_JRE_DISABLEDSCHEMES);
}
}
}
}
public static void updateSvnkitConfigureFile(String srcFilePath, String destFilePath) {

View File

@@ -173,6 +173,7 @@ public class TalendProxySelector extends ProxySelector {
checkProxyManager(IProxyChangeEvent.PROXY_DATA_CHANGED);
proxyManager.addProxyChangeListener(event -> checkProxyManager(event.getChangeType()));
NetworkUtil.checkProxyAuthSupport();
initHostMap();
initRedirectList();
}
@@ -374,6 +375,7 @@ public class TalendProxySelector extends ProxySelector {
if (printProxyLog) {
ExceptionHandler.log("TalendProxySelector.select " + uri);
}
NetworkUtil.checkProxyAuthSupport();
if (uri == null) {
List<Proxy> result = new ArrayList<>();
result.add(Proxy.NO_PROXY);

View File

@@ -24,6 +24,8 @@ package org.talend.commons.utils.system;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.eclipse.core.runtime.Platform;
/**
* Creates and updates properties for the eclipse commandline in case of relaunch <br/>
*
@@ -91,6 +93,23 @@ public class EclipseCommandLine {
updateOrCreateExitDataPropertyWithCommand(command, value, delete, false);
}
public static String getEclipseArgument(String argName) {
if (argName == null || argName.trim().isEmpty()) {
return null;
}
String[] commandLineArgs = Platform.getCommandLineArgs();
if (commandLineArgs == null || commandLineArgs.length <= 0) {
return null;
}
for (int i = 0; i < commandLineArgs.length - 1; i++) {
if (argName.equals(commandLineArgs[i])) {
return commandLineArgs[i + 1];
}
}
return null;
}
/**
* this creates or updates the org.eclipse.equinox.app.IApplicationContext.EXIT_DATA_PROPERTY by adding or changing
* the command with value, except if value is null then the command shall be removed.

View File

@@ -42,6 +42,7 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -50,11 +51,16 @@ import org.eclipse.core.resources.IContainer;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IFolder;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRunnable;
import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.OperationCanceledException;
import org.eclipse.core.runtime.jobs.ISchedulingRule;
import org.eclipse.core.runtime.jobs.Job;
import org.osgi.framework.Bundle;
import org.talend.commons.exception.CommonExceptionHandler;
import org.talend.commons.exception.ExceptionHandler;
@@ -1173,4 +1179,33 @@ public class FilesUtils {
}
}
public static void executeFolderAction(IProgressMonitor monitor, IResource parentFolder, IWorkspaceRunnable run)
throws CoreException {
if (Job.getJobManager().currentRule() == null) {
IWorkspace workspace = ResourcesPlugin.getWorkspace();
ISchedulingRule defaultRule = workspace.getRuleFactory().modifyRule(parentFolder);
ISchedulingRule noBlockRule = new ISchedulingRule() {
@Override
public boolean isConflicting(ISchedulingRule rule) {
return this.contains(rule);
}
@Override
public boolean contains(ISchedulingRule rule) {
if (this.equals(rule)) {
return true;
}
if (defaultRule.contains(rule)) {
return true;
}
return false;
}
};
workspace.run(run, noBlockRule, IWorkspace.AVOID_UPDATE, monitor);
} else {
run.run(monitor);
}
}
}

View File

@@ -575,7 +575,7 @@ public class ContextUtils {
return false;
}
// need check the raw value, because in sourceParam, it's raw
if (!sourceParam.getValue().equals(targetParamType.getRawValue())) {
if (!StringUtils.equals(sourceParam.getValue(), targetParamType.getRawValue())) {
return false;
}
@@ -1112,3 +1112,4 @@ public class ContextUtils {
return list;
}
}

View File

@@ -20,6 +20,7 @@ import java.util.Set;
import org.eclipse.core.runtime.IProgressMonitor;
import org.talend.commons.exception.BusinessException;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.IService;
import org.talend.core.model.general.ModuleNeeded.ELibraryInstallStatus;
import org.talend.core.model.process.IElement;
@@ -133,4 +134,14 @@ public interface ILibrariesService extends IService {
public void afterChangingLibraries();
}
public static ILibrariesService get() {
GlobalServiceRegister gsr = GlobalServiceRegister.getDefault();
if (gsr != null) {
if (gsr.isServiceRegistered(ILibrariesService.class)) {
return gsr.getService(ILibrariesService.class);
}
}
return null;
}
}

View File

@@ -113,8 +113,6 @@ public enum EParameterFieldType {
GROUPING_CAMPAIGN_CHOOSER, // htyin added for DQ Matching components
MULTI_PATTERN, // yyin added TDQ-13437
SYNC_NEXUS_BUTTON,
CHECK_NEXUS_BUTTON,
PATTERN_PROPERTY, // yyin, added TDQ-13437
UNIFIED_COMPONENTS,

View File

@@ -1003,7 +1003,9 @@ public final class ProcessUtils {
}
return false;
}
public static boolean isChildRouteProcess(IProcess process) {
List n = process.getNodesOfType("tRouteInput");
if (n!=null && n.size()!=0) {

View File

@@ -320,24 +320,28 @@ public class RepositoryUpdateManagerHelper {
// all the jobs
for (IRepositoryViewObject process : processRep) {
Item item = process.getProperty().getItem();
boolean found = false;
for (IProcess2 open : openedProcessList) {
if (open.getId().equals(item.getProperty().getId())) {
found = true;
try {
Item item = process.getProperty().getItem();
boolean found = false;
for (IProcess2 open : openedProcessList) {
if (open.getId().equals(item.getProperty().getId())) {
found = true;
}
}
}
if (found) {
continue;
}
checkMonitorCanceled(parentMonitor);
parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
if (found) {
continue;
}
checkMonitorCanceled(parentMonitor);
parentMonitor.subTask(RepositoryUpdateManager.getUpdateJobInfor(item.getProperty()));
// List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
// List<UpdateResult> resultFromProcess = getResultFromProcess(process, types, onlySimpleShow);
List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
if (resultFromProcess != null) {
resultList.addAll(resultFromProcess);
List<UpdateResult> resultFromProcess = getUpdatesNeededFromItems(parentMonitor, item, types);
if (resultFromProcess != null) {
resultList.addAll(resultFromProcess);
}
} catch (Exception ex) {
ExceptionHandler.process(ex);
}
parentMonitor.worked(1);
}

View File

@@ -19,7 +19,9 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
@@ -53,6 +55,14 @@ public class NexusServerUtils {
// the max search result is 200 by defult from nexus
private static final int MAX_SEARCH_COUNT = 200;
public static final Set<String> IGNORED_TYPES = new HashSet<String>();
static {
IGNORED_TYPES.add("pom");
IGNORED_TYPES.add("sha1");
IGNORED_TYPES.add("md5");
}
/**
*
* DOC check if the repository exist or not
@@ -236,36 +246,36 @@ public class NexusServerUtils {
private static int readDocument(Document document, List<MavenArtifact> artifacts) throws Exception {
List<Node> list = document.selectNodes("/searchNGResponse/data/artifact");//$NON-NLS-1$
for (Node arNode : list) {
MavenArtifact artifact = new MavenArtifact();
artifacts.add(artifact);
artifact.setGroupId(arNode.selectSingleNode("groupId").getText());//$NON-NLS-1$
artifact.setArtifactId(arNode.selectSingleNode("artifactId").getText());//$NON-NLS-1$
artifact.setVersion(arNode.selectSingleNode("version").getText());//$NON-NLS-1$
Node descNode = arNode.selectSingleNode("description");//$NON-NLS-1$
if (descNode != null) {
artifact.setDescription(descNode.getText());
}
Node urlNode = arNode.selectSingleNode("url");//$NON-NLS-1$
if (urlNode != null) {
artifact.setUrl(urlNode.getText());
}
Node licenseNode = arNode.selectSingleNode("license");//$NON-NLS-1$
if (licenseNode != null) {
artifact.setLicense(licenseNode.getText());
}
Node licenseUrlNode = arNode.selectSingleNode("licenseUrl");//$NON-NLS-1$
if (licenseUrlNode != null) {
artifact.setLicenseUrl(licenseUrlNode.getText());
}
List<Node> artLinks = arNode.selectNodes("artifactHits/artifactHit/artifactLinks/artifactLink");//$NON-NLS-1$
for (Node link : artLinks) {
MavenArtifact artifact = new MavenArtifact();
artifact.setGroupId(arNode.selectSingleNode("groupId").getText());//$NON-NLS-1$
artifact.setArtifactId(arNode.selectSingleNode("artifactId").getText());//$NON-NLS-1$
artifact.setVersion(arNode.selectSingleNode("version").getText());//$NON-NLS-1$
Node descNode = arNode.selectSingleNode("description");//$NON-NLS-1$
if (descNode != null) {
artifact.setDescription(descNode.getText());
}
Node urlNode = arNode.selectSingleNode("url");//$NON-NLS-1$
if (urlNode != null) {
artifact.setUrl(urlNode.getText());
}
Node licenseNode = arNode.selectSingleNode("license");//$NON-NLS-1$
if (licenseNode != null) {
artifact.setLicense(licenseNode.getText());
}
Node licenseUrlNode = arNode.selectSingleNode("licenseUrl");//$NON-NLS-1$
if (licenseUrlNode != null) {
artifact.setLicenseUrl(licenseUrlNode.getText());
}
Node extensionElement = link.selectSingleNode("extension");//$NON-NLS-1$
String extension = null;
String classifier = null;
if (extensionElement != null) {
if ("pom".equals(extensionElement.getText())) {//$NON-NLS-1$
if (IGNORED_TYPES.contains(extensionElement.getText())) {// $NON-NLS-1$
continue;
}
extension = extensionElement.getText();
@@ -276,6 +286,7 @@ public class NexusServerUtils {
}
artifact.setType(extension);
artifact.setClassifier(classifier);
artifacts.add(artifact);
}
}
return list.size();

View File

@@ -47,6 +47,8 @@ public class RepositoryConstants {
public static final String JOBLET_DOCUMENTATION_PATH = "documentations/generated/joblets"; //$NON-NLS-1$
public static final String PROJECT_SETTINGS_CUSTOM_URI_MAP = "custom_uri_mapping.json"; //$NON-NLS-1$
public static final String SYSTEM_DIRECTORY = "system"; //$NON-NLS-1$
public static final String USER_DEFINED = "UserDefined"; //$NON-NLS-1$

View File

@@ -36,8 +36,6 @@ import org.talend.core.model.process.IElement;
import org.talend.core.model.process.IElementParameter;
import org.talend.core.model.process.INode;
import org.talend.core.model.process.IProcess2;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.core.ui.CoreUIPlugin;
import org.talend.core.ui.process.IGEFProcess;
import org.talend.core.ui.services.IDesignerCoreUIService;
@@ -192,33 +190,16 @@ public class ModuleListCellEditor extends DialogCellEditor {
}
// enable to refresh component setting after change modules.
IElement element = this.tableParam.getElement();
boolean isNotCConfig = element.getElementParameter("COMPONENT_NAME") == null ?
true : !"cConfig".equals(element.getElementParameter("COMPONENT_NAME").getValue());
if (element != null && isNotCConfig) {
if (element != null) {
IElementParameter updateComponentsParam = element.getElementParameter("UPDATE_COMPONENTS"); //$NON-NLS-1$
if (updateComponentsParam != null) {
updateComponentsParam.setValue(Boolean.TRUE);
}
}
// cConfig
if (!isNotCConfig) {
if (newValue.startsWith(MavenUrlHelper.MVN_PROTOCOL)) {
MavenArtifact art = MavenUrlHelper.parseMvnUrl(newValue);
newValue = art.getFileName();
}
}
//
executeCommand(new ModelChangeCommand(tableParam, param.getName(), newValue, index));
if (newVal != null) {
executeCommand(new ModelChangeCommand(tableParam, "JAR_PATH", newVal, index));
}
if (nexusVersion != null) {
executeCommand(new ModelChangeCommand(tableParam, "JAR_NEXUS_VERSION", nexusVersion, index));
}
oldValue = newValue;
if (getTableViewer() != null) {
getTableViewer().refresh(true);

View File

@@ -10,7 +10,7 @@
<artifactId>studio-tacokit-dependencies</artifactId>
<packaging>pom</packaging>
<properties>
<tacokit.components.version>1.14.0-SNAPSHOT</tacokit.components.version>
<tacokit.components.version>1.16.0-SNAPSHOT</tacokit.components.version>
</properties>
<repositories>
<repository>

View File

@@ -33,7 +33,7 @@
<artifactItem>
<groupId>org.talend.studio</groupId>
<artifactId>studio-maven-repository</artifactId>
<version>7.2.1.v20190226</version>
<version>7.4.1.v20201111</version>
<type>zip</type>
<overWrite>true</overWrite>
<outputDirectory>${project.basedir}/../repository</outputDirectory>

View File

@@ -18,6 +18,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.text.ParseException;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Properties;
@@ -39,6 +40,14 @@ public class CodeM2CacheManager {
private static final String KEY_SEPERATOR = "|"; //$NON-NLS-1$
private static final String EMPTY_DATE;
static {
Calendar c = Calendar.getInstance();
c.setTimeInMillis(0);
EMPTY_DATE = ResourceHelper.dateFormat().format(c.getTime());
}
public static boolean needUpdateCodeProject(Project project, ERepositoryObjectType codeType) {
try {
String projectTechName = project.getTechnicalLabel();
@@ -56,11 +65,11 @@ public class CodeM2CacheManager {
// check M
for (IRepositoryViewObject codeItem : allCodes) {
Property property = codeItem.getProperty();
String key = getCacheKey(projectTechName, property);
String cachedTimestamp = cache.getProperty(key);
if (cachedTimestamp != null) {
Date currentDate = ResourceHelper.dateFormat().parse(getCacheDate(projectTechName, property));
Date cachedDate = ResourceHelper.dateFormat().parse(cachedTimestamp);
String key = getKey(projectTechName, property);
String cacheValue = cache.getProperty(key);
if (cacheValue != null) {
Date currentDate = ResourceHelper.dateFormat().parse(getModifiedDate(projectTechName, property));
Date cachedDate = ResourceHelper.dateFormat().parse(cacheValue);
if (currentDate.compareTo(cachedDate) != 0) {
return true;
}
@@ -82,8 +91,8 @@ public class CodeM2CacheManager {
Properties cache = new Properties();
for (IRepositoryViewObject codeItem : allCodes) {
Property property = codeItem.getProperty();
String key = getCacheKey(projectTechName, property);
String value = getCacheDate(projectTechName, property);
String key = getKey(projectTechName, property);
String value = getModifiedDate(projectTechName, property);
cache.put(key, value);
}
cache.store(out, StringUtils.EMPTY);
@@ -98,12 +107,13 @@ public class CodeM2CacheManager {
return new File(MavenPlugin.getMaven().getLocalRepositoryPath(), cacheFileName);
}
private static String getCacheKey(String projectTechName, Property property) {
private static String getKey(String projectTechName, Property property) {
return projectTechName + KEY_SEPERATOR + property.getId() + KEY_SEPERATOR + property.getVersion(); // $NON-NLS-1$
}
private static String getCacheDate(String projectTechName, Property property) {
return (String) property.getAdditionalProperties().get(ItemProductKeys.DATE.getModifiedKey());
private static String getModifiedDate(String projectTechName, Property property) {
String modifiedDate = (String) property.getAdditionalProperties().get(ItemProductKeys.DATE.getModifiedKey());
return StringUtils.isNotBlank(modifiedDate) ? modifiedDate : EMPTY_DATE;
}
}

View File

@@ -45,6 +45,7 @@ import org.talend.core.model.utils.JavaResourcesHelper;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenConstants;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.core.runtime.process.LastGenerationInfo;
import org.talend.core.runtime.process.TalendProcessArgumentConstant;
import org.talend.core.runtime.projectsetting.IProjectSettingTemplateConstants;
import org.talend.core.runtime.repository.build.IMavenPomCreator;
@@ -52,6 +53,7 @@ import org.talend.core.ui.ITestContainerProviderService;
import org.talend.designer.maven.model.TalendMavenConstants;
import org.talend.designer.maven.template.ETalendMavenVariables;
import org.talend.designer.maven.tools.ProcessorDependenciesManager;
import org.talend.designer.maven.utils.JobUtils;
import org.talend.designer.maven.utils.PomIdsHelper;
import org.talend.designer.maven.utils.PomUtil;
import org.talend.designer.runprocess.IBigDataProcessor;
@@ -134,9 +136,16 @@ public abstract class AbstractMavenProcessorPom extends CreateMavenBundleTemplat
Map<ETalendMavenVariables, String> variablesValuesMap = new HashMap<ETalendMavenVariables, String>();
// no need check property is null or not, because if null, will get default ids.
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
variablesValuesMap.put(ETalendMavenVariables.JobArtifactId, PomIdsHelper.getJobArtifactId(property));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));
if (JobUtils.isJob(property) && ProcessUtils.isChildRouteProcess(process)) {
JobInfo lastMainJob = LastGenerationInfo.getInstance().getLastMainJob();
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(lastMainJob.getProcessor().getProperty()));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(lastMainJob.getProcessor().getProperty()));
}else {
variablesValuesMap.put(ETalendMavenVariables.JobGroupId, PomIdsHelper.getJobGroupId(property));
variablesValuesMap.put(ETalendMavenVariables.JobVersion, PomIdsHelper.getJobVersion(property));
}
variablesValuesMap.put(ETalendMavenVariables.JobArtifactId, PomIdsHelper.getJobArtifactId(property));
variablesValuesMap.put(ETalendMavenVariables.TalendJobVersion, property.getVersion());
final String jobName = JavaResourcesHelper.escapeFileName(process.getName());
variablesValuesMap.put(ETalendMavenVariables.JobName, jobName);

View File

@@ -19,6 +19,7 @@ import java.util.Set;
import org.talend.commons.exception.PersistenceException;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.model.process.INode;
import org.talend.core.model.process.IProcess;
import org.talend.core.model.process.JobInfo;
import org.talend.core.model.process.ProcessUtils;
@@ -99,6 +100,13 @@ public class JobUtils {
}
return clonedJobInfos;
}
public static boolean isRoute(Property p) {
if (p != null) {
return ERepositoryObjectType.getType(p).equals(ERepositoryObjectType.PROCESS_ROUTE);
}
return false;
}
public static boolean isJob(JobInfo job) {
if (job != null && job.getProcessItem() != null) {

View File

@@ -64,7 +64,6 @@ import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.jobs.ISchedulingRule;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.jobs.MultiRule;
import org.eclipse.m2e.core.MavenPlugin;
import org.eclipse.m2e.core.embedder.MavenModelManager;
@@ -73,6 +72,7 @@ import org.talend.commons.exception.PersistenceException;
import org.talend.commons.runtime.utils.io.IOUtils;
import org.talend.commons.utils.VersionUtils;
import org.talend.commons.utils.generation.JavaUtils;
import org.talend.commons.utils.io.FilesUtils;
import org.talend.commons.utils.workbench.resources.ResourceUtils;
import org.talend.core.GlobalServiceRegister;
import org.talend.core.ILibraryManagerService;
@@ -583,7 +583,7 @@ public class PomUtil {
IProject fsProject = ResourceUtils.getProject(project);
IFolder tmpFolder = fsProject.getFolder("temp");
if (!tmpFolder.exists()) {
executeFolderAction(monitor, fsProject, new IWorkspaceRunnable() {
FilesUtils.executeFolderAction(monitor, fsProject, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -595,7 +595,7 @@ public class PomUtil {
createTempFile.delete();
String tmpFolderName = createTempFile.getName();
IFolder folder = tmpFolder.getFolder(tmpFolderName);
executeFolderAction(monitor, tmpFolder, new IWorkspaceRunnable() {
FilesUtils.executeFolderAction(monitor, tmpFolder, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -604,7 +604,7 @@ public class PomUtil {
});
IFile pomFile = folder.getFile(TalendMavenConstants.POM_FILE_NAME);
executeFolderAction(monitor, folder, new IWorkspaceRunnable() {
FilesUtils.executeFolderAction(monitor, folder, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
@@ -622,35 +622,6 @@ public class PomUtil {
return null;
}
private static void executeFolderAction(IProgressMonitor monitor, IResource parentFolder, IWorkspaceRunnable run)
throws CoreException {
if (Job.getJobManager().currentRule() == null) {
IWorkspace workspace = ResourcesPlugin.getWorkspace();
ISchedulingRule defaultRule = workspace.getRuleFactory().modifyRule(parentFolder);
ISchedulingRule noBlockRule = new ISchedulingRule() {
@Override
public boolean isConflicting(ISchedulingRule rule) {
return this.contains(rule);
}
@Override
public boolean contains(ISchedulingRule rule) {
if (this.equals(rule)) {
return true;
}
if (defaultRule.contains(rule)) {
return true;
}
return false;
}
};
workspace.run(run, noBlockRule, IWorkspace.AVOID_UPDATE, monitor);
} else {
run.run(monitor);
}
}
private static Model createModel(MavenArtifact artifact) {
Model pomModel = new Model();
pomModel.setModelVersion(TalendMavenConstants.POM_VERSION);

View File

@@ -38,7 +38,6 @@ import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.librariesmanager.model.ModulesNeededProvider;
import org.talend.librariesmanager.nexus.utils.VersionUtil;
import org.talend.librariesmanager.ui.LibManagerUiPlugin;
import org.talend.repository.ProjectManager;
/*
* Created by bhe on Sep 3, 2020

View File

@@ -13,25 +13,38 @@
package org.talend.librariesmanager.model.service;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Set;
import net.sf.json.JSONObject;
import org.codehaus.jackson.map.ObjectMapper;
import org.eclipse.core.resources.IFile;
import org.eclipse.core.resources.IFolder;
import org.eclipse.core.resources.IProject;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspaceRunnable;
import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.talend.commons.exception.ExceptionHandler;
import org.talend.commons.exception.LoginException;
import org.talend.commons.exception.PersistenceException;
import org.talend.commons.utils.io.FilesUtils;
import org.talend.commons.utils.workbench.resources.ResourceUtils;
import org.talend.core.model.general.Project;
import org.talend.core.runtime.CoreRuntimePlugin;
import org.talend.repository.ProjectManager;
import org.talend.repository.RepositoryWorkUnit;
import org.talend.repository.model.IProxyRepositoryFactory;
import org.talend.repository.model.RepositoryConstants;
import net.sf.json.JSONObject;
/**
* created by wchen on Aug 18, 2017 Detailled comment
@@ -43,17 +56,30 @@ public class CustomUriManager {
private static CustomUriManager manager = new CustomUriManager();;
private static final String CUSTOM_URI_MAP = "custom_uri_mapping.json";
private static long lastModified = 0;
private static boolean isNeedReload = false;
private final Object reloadingLock = new Object();
private CustomUriManager() {
IFolder resourcePath = getResourcePath();
try {
customURIObject = loadResources(getResourcePath(), CUSTOM_URI_MAP);
Assert.isNotNull(resourcePath);
customURIObject = loadResources(resourcePath, RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
} catch (Exception e) {
ExceptionHandler.process(e);
try {
if (resourcePath != null) {
customURIObject = loadResources(resourcePath.getLocation().toPortableString(),
RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
}
} catch (Exception e1) {
ExceptionHandler.process(e1);
/**
* Seems dangrous to load an empty json here, because it may overwrite the existing one
*/
// customURIObject = new JSONObject();
}
}
}
@@ -61,34 +87,121 @@ public class CustomUriManager {
return manager;
}
private synchronized JSONObject loadResources(String path, String fileName) throws IOException {
BufferedReader br = null;
private JSONObject loadResources(String path, String fileName) throws IOException {
JSONObject jsonObj = new JSONObject();
InputStream in = null;
try {
File file = new File(path, fileName);
if (file.exists()) {
br = new BufferedReader(new FileReader(file));
StringBuffer buffer = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null) {
buffer.append(line);
}
jsonObj = JSONObject.fromObject(buffer.toString());
jsonObj = loadResources(new FileInputStream(file), fileName);
}
} finally {
if (br != null) {
br.close();
if (in != null) {
try {
in.close();
} catch (Exception e) {
}
}
}
return jsonObj;
}
private JSONObject loadResources(IFolder path, String fileName) throws CoreException, IOException {
JSONObject jsonObj = new JSONObject();
InputStream in = null;
try {
IFile file = path.getFile(fileName);
FilesUtils.executeFolderAction(new NullProgressMonitor(), path, new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
file.refreshLocal(IResource.DEPTH_ZERO, monitor);
}
});
if (file.isAccessible()) {
in = file.getContents(true);
return loadResources(in, fileName);
}
} finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e) {
}
}
return jsonObj;
}
private JSONObject loadResources(InputStream in, String fileName) throws IOException {
BufferedReader br = null;
JSONObject jsonObj = new JSONObject();
try {
br = new BufferedReader(new InputStreamReader(in));
StringBuffer buffer = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null) {
buffer.append(line);
}
jsonObj = JSONObject.fromObject(buffer.toString());
} finally {
if (br != null) {
try {
br.close();
} catch (Exception e) {
}
}
}
return jsonObj;
}
private void saveResource(JSONObject customMap, IFolder filePath, String fileName, boolean isExport) {
ByteArrayOutputStream out = null;
ByteArrayInputStream in = null;
try {
IFile file = filePath.getFile(fileName);
out = new ByteArrayOutputStream();
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(out, customMap);
in = new ByteArrayInputStream(out.toByteArray());
final InputStream fin = in;
FilesUtils.executeFolderAction(new NullProgressMonitor(), file.getParent(), new IWorkspaceRunnable() {
@Override
public void run(IProgressMonitor monitor) throws CoreException {
file.refreshLocal(IResource.DEPTH_ZERO, monitor);
if (!file.exists()) {
file.create(fin, false, monitor);
} else {
file.setContents(fin, true, false, monitor);
}
}
});
} catch (Exception e) {
ExceptionHandler.process(e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException ex) {
// ExceptionHandler.process(ex);
}
}
if (in != null) {
try {
in.close();
} catch (IOException ex) {
// ExceptionHandler.process(ex);
}
}
}
}
private void saveResource(JSONObject customMap, String filePath, String fileName, boolean isExport) {
try {
File file = new File(filePath, fileName);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.writerWithDefaultPrettyPrinter().writeValue(file, customMap);
lastModified = file.lastModified();
} catch (IOException e) {
ExceptionHandler.process(e);
}
@@ -100,7 +213,7 @@ public class CustomUriManager {
@Override
public void run() throws PersistenceException, LoginException {
saveResource(customURIObject, getResourcePath(), CUSTOM_URI_MAP, false);
saveResource(customURIObject, getResourcePath(), RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP, false);
}
};
IProxyRepositoryFactory factory = CoreRuntimePlugin.getInstance().getProxyRepositoryFactory();
@@ -111,12 +224,11 @@ public class CustomUriManager {
}
private String getResourcePath() {
private IFolder getResourcePath() {
try {
Project currentProject = ProjectManager.getInstance().getCurrentProject();
IProject project = ResourceUtils.getProject(currentProject);
IFolder settingsFolder = project.getFolder(".settings");
return settingsFolder.getLocation().toPortableString();
return project.getFolder(".settings");
} catch (PersistenceException e) {
ExceptionHandler.process(e);
}
@@ -124,7 +236,11 @@ public class CustomUriManager {
}
public void put(String key, String value) {
reloadCustomMapping();
try {
reloadCustomMapping();
} catch (Exception e) {
ExceptionHandler.process(e);
}
if (value != null) {
customURIObject.put(key, value);
} else {
@@ -133,7 +249,11 @@ public class CustomUriManager {
}
public String get(String key) {
reloadCustomMapping();
try {
reloadCustomMapping();
} catch (Exception e) {
ExceptionHandler.process(e);
}
if (customURIObject.containsKey(key)) {
return customURIObject.getString(key);
}
@@ -159,16 +279,22 @@ public class CustomUriManager {
public void reloadCustomMapping() {
try {
File file = new File(getResourcePath(), CUSTOM_URI_MAP);
long modifyDate = file.lastModified();
if (isNeedReload || modifyDate > lastModified) {
customURIObject.clear();
JSONObject loadResources = loadResources(getResourcePath(), CUSTOM_URI_MAP);
customURIObject.putAll(loadResources);
lastModified = modifyDate;
isNeedReload = false;
if (isNeedReload || customURIObject == null) {
synchronized (reloadingLock) {
if (isNeedReload || customURIObject == null) {
if (customURIObject == null) {
customURIObject = new JSONObject();
} else {
customURIObject.clear();
}
JSONObject loadResources = loadResources(getResourcePath(),
RepositoryConstants.PROJECT_SETTINGS_CUSTOM_URI_MAP);
customURIObject.putAll(loadResources);
isNeedReload = false;
}
}
}
} catch (IOException e) {
} catch (Exception e) {
ExceptionHandler.process(e);
}
}

View File

@@ -41,6 +41,7 @@ import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.designer.maven.aether.RepositorySystemFactory;
import org.talend.librariesmanager.i18n.Messages;
import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
import org.talend.utils.sugars.TypedReturnCode;
import net.sf.json.JSONArray;
@@ -259,6 +260,9 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
artifact.setVersion(v);
artifact.setType(type);
artifact.setLastUpdated(lastUpdated);
String regex = a + "-" + v;
String classifier = ShareLibrariesUtil.getMavenClassifier(artifactPath, regex, type);
artifact.setClassifier(classifier);
fillChecksumData(jsonObject, artifact);
resultList.add(artifact);
}
@@ -353,6 +357,9 @@ public class ArtifacoryRepositoryHandler extends AbstractArtifactRepositoryHandl
artifact.setVersion(v);
artifact.setType(type);
artifact.setLastUpdated(lastUpdated);
String regex = a + "-" + v;
String classifier = ShareLibrariesUtil.getMavenClassifier(artifactPath, regex, type);
artifact.setClassifier(classifier);
fillChecksumData(jsonObject, artifact);
resultList.add(artifact);
}

View File

@@ -16,9 +16,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
@@ -32,21 +30,18 @@ import org.apache.log4j.Logger;
import org.eclipse.core.runtime.IProgressMonitor;
import org.talend.core.nexus.ArtifactRepositoryBean;
import org.talend.core.nexus.HttpClientTransport;
import org.talend.core.nexus.NexusServerUtils;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.librariesmanager.nexus.utils.ShareLibrariesUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
private static Logger log = Logger.getLogger(AbsNexus3SearchHandler.class);
protected ArtifactRepositoryBean serverBean;
private static final Set<String> IGNORED_TYPES = new HashSet<String>();
static {
IGNORED_TYPES.add("pom");
IGNORED_TYPES.add("sha1");
IGNORED_TYPES.add("md5");
}
private static Logger log = Logger.getLogger(AbsNexus3SearchHandler.class);
protected ArtifactRepositoryBean serverBean;
/**
* {@value}
@@ -135,37 +130,53 @@ public abstract class AbsNexus3SearchHandler implements INexus3SearchHandler {
if (resultArray != null) {
for (int i = 0; i < resultArray.size(); i++) {
JSONObject jsonObject = resultArray.getJSONObject(i);
MavenArtifact artifact = new MavenArtifact();
artifact.setGroupId(jsonObject.getString("group")); //$NON-NLS-1$
artifact.setArtifactId(jsonObject.getString("name")); //$NON-NLS-1$
artifact.setVersion(jsonObject.getString("version")); //$NON-NLS-1$
JSONArray assertsArray = jsonObject.getJSONArray("assets"); //$NON-NLS-1$
artifact.setType(getPackageType(assertsArray));
if (artifact.getType() != null) {
fillCheckSumData(assertsArray, artifact);
resultList.add(artifact);
String repository = jsonObject.getString("repository");//$NON-NLS-1$
String group = jsonObject.getString("group");//$NON-NLS-1$
String name = jsonObject.getString("name");//$NON-NLS-1$
String version = jsonObject.getString("version");//$NON-NLS-1$
JSONArray assertsArray = jsonObject.getJSONArray("assets");//$NON-NLS-1$
if (assertsArray != null) {
for (int j = 0; j < assertsArray.size(); j++) {
JSONObject assertsObject = assertsArray.getJSONObject(j);
String packageType = getPackageType(assertsObject);
if (packageType != null) {
MavenArtifact artifact = new MavenArtifact();
String path = assertsObject.getString("path"); //$NON-NLS-1$
String regex = name + "-" + version;
String classifier = ShareLibrariesUtil.getMavenClassifier(path, regex, packageType);
artifact.setGroupId(group);
artifact.setArtifactId(name);
artifact.setVersion(version);
artifact.setType(packageType);
artifact.setClassifier(classifier);
if (artifact.getType() != null) {
fillCheckSumData(assertsArray, artifact);
resultList.add(artifact);
}
}
}
}
}
}
return continuationToken;
}
protected String getPackageType(JSONArray assertsArray) {
protected String getPackageType(JSONObject jsonObject) {
String type = null;
if (assertsArray != null) {
for (int i = 0; i < assertsArray.size(); i++) {
JSONObject jsonObject = assertsArray.getJSONObject(i);
String path = jsonObject.getString("path"); //$NON-NLS-1$
int idx = path.lastIndexOf('.');
if (idx > -1) {
type = path.substring(idx + 1);
}
if (!IGNORED_TYPES.contains(type)) {
return type;
}
}
String path = jsonObject.getString("path"); //$NON-NLS-1$
int idx = path.lastIndexOf('.');
if (idx > -1) {
type = path.substring(idx + 1);
}
if (!NexusServerUtils.IGNORED_TYPES.contains(type)) {
return type;
}
return null;
}

View File

@@ -123,4 +123,17 @@ public class ShareLibrariesUtil {
}
return sb.toString();
}
public static String getMavenClassifier(String path, String regex, String packageType) {
String classifier = null;
// javax/xml/bind/acxb-test/2.2.6/acxb-test-2.2.6-jdk10.dll
path = StringUtils.removeEnd(path, "." + packageType);
// javax/xml/bind/acxb-test/2.2.6/acxb-test-2.2.6-jdk10
path = StringUtils.substringAfter(path, regex);// -jdk10
path = StringUtils.stripStart(path, "-");// jdk10
if (StringUtils.isNotBlank(path)) {
classifier = path;
}
return classifier;
}
}

View File

@@ -8,5 +8,78 @@
<relativePath>../../../</relativePath>
</parent>
<artifactId>org.talend.librariesmanager.test</artifactId>
<packaging>eclipse-plugin</packaging>
</project>
<packaging>eclipse-plugin</packaging>
<dependencies>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>dll</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
<type>exe</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.new</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>zip</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>dll</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
<type>exe</type>
</dependency>
<dependency>
<groupId>org.talend.studio.test</groupId>
<artifactId>nexus.upload.test.old</artifactId>
<version>1.0.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>generate-sources</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeGroupIds>org.talend.studio.test</includeGroupIds>
<copyPom>true</copyPom>
<includeArtifactIdIds>nexus.upload.test.old,nexus.upload.test.new</includeArtifactIdIds>
<excludeTransitive>true</excludeTransitive>
<outputDirectory>${project.basedir}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,5 @@
rep.create.endpoint=/service/rest/v1/repositories/maven/hosted
rep.list.endpoint=/service/rest/v1/repositories/maven/hosted/{repid}
component.search.endpoint=/service/rest/v1/search
component.delete.endpoint=/service/rest/v1/components/{id}
component.upload.command=curl -v -u {user}:{password} -X POST \"{server}/service/rest/v1/components?repository={repid}\" -F maven2.groupId={groupid} -F maven2.artifactId={artifactid} -F maven2.version={version} -F maven2.asset1=@{filepath} -F maven2.asset1.extension={fileext}

View File

@@ -0,0 +1,18 @@
{
"name": "NEWREPNAME",
"online": true,
"storage": {
"blobStoreName": "default",
"strictContentTypeValidation": false,
"writePolicy": "ALLOW"
},
"cleanup": {
"policyNames": [
"string"
]
},
"maven": {
"versionPolicy": "NEWVERSIONPOLICY",
"layoutPolicy": "STRICT"
}
}

View File

@@ -12,17 +12,72 @@
// ============================================================================
package org.talend.librariesmanager.nexus;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Dictionary;
import java.util.Hashtable;
import java.util.List;
import java.util.Properties;
import org.apache.commons.codec.digest.DigestUtils;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.Platform;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import org.osgi.framework.InvalidSyntaxException;
import org.osgi.framework.ServiceReference;
import org.osgi.service.cm.Configuration;
import org.osgi.service.cm.ConfigurationAdmin;
import org.talend.core.nexus.ArtifactRepositoryBean;
import org.talend.core.nexus.IRepositoryArtifactHandler;
import org.talend.core.nexus.NexusServerUtils;
import org.talend.core.nexus.RepositoryArtifactHandlerManager;
import org.talend.core.nexus.TalendMavenResolver;
import org.talend.core.nexus.ArtifactRepositoryBean.NexusType;
import org.talend.core.runtime.CoreRuntimePlugin;
import org.talend.core.runtime.maven.MavenArtifact;
import org.talend.core.runtime.maven.MavenUrlHelper;
import org.talend.librariesmanager.model.service.LocalLibraryManager;
import org.talend.librariesmanager.nexus.utils.NexusServerManagerProxy;
import org.talend.librariesmanager.nexus.utils.RestAPIUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
/**
* created by wchen on Aug 18, 2017 Detailled comment
*
*/
public class Nexus3RepositoryHandlerTest {
private static Properties nexusprops = new Properties();
private static ArtifactRepositoryBean customNexusServer;
private static IRepositoryArtifactHandler repHandler;
private static String[] types = new String[] {"jar", "pom", "exe", "zip", "dll"};
@BeforeClass
public static void init() throws FileNotFoundException, IOException {
URL entry = Platform.getBundle("org.talend.librariesmanager.test").getEntry("resources/nexus/nexus3.properties");
nexusprops.load(new FileInputStream(FileLocator.toFileURL(entry).getFile()));
customNexusServer = NexusServerManagerProxy.getInstance().getCustomNexusServer();
repHandler = RepositoryArtifactHandlerManager.getRepositoryHandler(customNexusServer);
createNexusRepository(customNexusServer.getRepositoryId(),"RELEASE");
createNexusRepository(customNexusServer.getSnapshotRepId(),"SNAPSHOT");
}
@Test
public void testGetRepositoryURL() {
ArtifactRepositoryBean serverBean = new ArtifactRepositoryBean();
@@ -37,5 +92,329 @@ public class Nexus3RepositoryHandlerTest {
String snapshotUrl = handler.getRepositoryURL(false);
Assert.assertEquals("http://localhost:8081/repository/snapshot-repository/", snapshotUrl);
}
@Test
public void testCheckConnectionOK() {
Assert.assertTrue(repHandler.checkConnection());
}
@Test
public void testCheckConnectionFalse() {
ArtifactRepositoryBean dummyNexusServer = NexusServerManagerProxy.getInstance().getDummyNexusServer();
IRepositoryArtifactHandler repHandler = RepositoryArtifactHandlerManager.getRepositoryHandler(dummyNexusServer);
Assert.assertFalse(repHandler.checkConnection());
}
@Test
public void testDeployOnSnapshot() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/" + type;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
// deploy the file to nexus
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
File reolved = repHandler.resolve(artifact);
assertEquals(originalSHA1,getSha1(reolved));
} finally {
deleteNexusArtifact(artifact, false, true);
}
}
}
@Test
public void testDeployOnRelease() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0/" + type ;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
// deploy file to nexus
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type , artifact.getVersion());
File resolved = repHandler.resolve(artifact);
assertEquals(originalSHA1,getSha1(resolved));
} finally {
deleteNexusArtifact(artifact, true, true);
}
}
}
@Test
public void testUpdateOnSnapshot() throws Exception {
for ( String type : types ) {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/" + type ;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
clearLocalFile(uri);
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File originalJarFile = new File(FileLocator.toFileURL(entry).getFile());
entry = bundle.getEntry("/lib/nexus.upload.test.new-1.0.0." + type);
File newJarFile = new File(FileLocator.toFileURL(entry).getFile());
String originalSHA1 = getSha1(originalJarFile);
String newJarSHA1 = getSha1(newJarFile);
// deploy original jar without resolving
repHandler.deploy(originalJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
//deploy new jar
repHandler.deploy(newJarFile, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), type, artifact.getVersion());
//resolve and check the local file
File resolved = repHandler.resolve(artifact);
assertEquals(newJarSHA1,getSha1(resolved));
} finally {
deleteNexusArtifact(artifact, false, true);
}
}
}
@Test
public void testResolveSha1NotExist() throws Exception {
String uri = "mvn:org.talend.libraries/not-existing/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
File f = repHandler.resolve(artifact);
assertNull(f);
} catch (FileNotFoundException fnfe) {
//It is one expected exception.
return;
}
}
@Test
public void testSearchOnRelease() throws Exception {
for (String type: types) {
String uri = "mvn:org.talend.libraries/test/6.0.0/" + type;
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0." + type);
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Search empty
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
if ( searchRet != null && searchRet.size() > 0 ) {
Assert.fail("The artifact:" + uri + " is not expected to intalled before the test");
}
//Step3:Deploy the artifact
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), artifact.getType(), artifact.getVersion());
//Step4:Verify the search result
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
int i = 30;
while ( searchRet.size() < 1 && i > 0 ) {
Thread.sleep(1000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), true, false);
i--;
}
if (NexusServerUtils.IGNORED_TYPES.contains(type)) {
assertEquals("Should get 0 artifact when searching for ignored type of " + uri , 0, searchRet.size());
} else {
assertEquals("Should get 1 artifact when searching for " + uri , 1, searchRet.size());
assertEquals(artifact.getArtifactId(), searchRet.get(0).getArtifactId());
assertEquals(artifact.getGroupId(), searchRet.get(0).getGroupId());
assertEquals(artifact.getVersion(), searchRet.get(0).getVersion());
assertEquals(artifact.getClassifier(), searchRet.get(0).getClassifier());
assertEquals(artifact.getType(), searchRet.get(0).getType());
}
} finally {
deleteNexusArtifact(artifact, true, true);
Thread.sleep(1000);
}
}
}
@Test
public void testSearchOnSnapshot() throws Exception {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0.jar");
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Search empty
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
if ( searchRet != null && searchRet.size() > 0 ) {
Assert.fail("The artifact is not expected to intalled before the test");
}
//Step3:Deploy the artifact
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), "jar", artifact.getVersion());
//Step4:Verify the search result
//In snapshot repository, Timestamp is added in item version like 6.0.0-20201102.054042-1,
//the search result for given version like 6.0.0 will be empty. Currently the version will not be given in search criteria.
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
int i = 30;
while ( searchRet.size() < 1 && i > 0 ) {
Thread.sleep(1000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i--;
}
debug("testSearchSnapshot:" + searchRet);
assertEquals(1, searchRet.size());
assertEquals(artifact.getArtifactId(), searchRet.get(0).getArtifactId());
assertEquals(artifact.getGroupId(), searchRet.get(0).getGroupId());
//assertEquals(artifact.getVersion(), searchRet.get(0).getVersion());
assertEquals(artifact.getClassifier(), searchRet.get(0).getClassifier());
assertEquals(artifact.getType(), searchRet.get(0).getType());
} finally {
deleteNexusArtifact(artifact, false, true);
Thread.sleep(1000);
}
}
@Test
public void testPaginationSearchOnSnapshot() throws Exception {
//Pagination is enabled by Nexus when a search returns 50 results
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/zip";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
List<MavenArtifact> searchRet = new ArrayList<MavenArtifact>();
try {
//Step1:Prepare the installed artifact
Bundle bundle = Platform.getBundle("org.talend.librariesmanager.test");
final URL entry = bundle.getEntry("/lib/nexus.upload.test.old-1.0.0.zip");
File fileToBeInstalled = new File(FileLocator.toFileURL(entry).getFile());
//Step2:Deploy the artifacts for 101 times on snapshot repository .
int i = 51;
while ( i> 0 ) {
repHandler.deploy(fileToBeInstalled, artifact.getGroupId(), artifact.getArtifactId(), artifact.getClassifier(), "zip", artifact.getVersion());
i--;
}
//Step3:Verify the search result. It will take some time for the deployment and wait for most 60 seconds
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i = 6;
while ( searchRet.size() < 51 && i > 0 ) {
Thread.sleep(10000);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
i--;
}
assertEquals(51, searchRet.size());
} finally {
deleteNexusArtifacts(searchRet, false, true);
}
}
//@Test
public void testUpdateMavenResolver() throws InvalidSyntaxException, IOException {
Dictionary<String, String> props = new Hashtable<String, String>();
props.put("org.ops4j.pax.url.mvn.socket.readTimeout", "59");
props.put("org.ops4j.pax.url.mvn.socket.connectionTimeout", "119");
repHandler.updateMavenResolver(TalendMavenResolver.TALEND_ARTIFACT_LIBRARIES_RESOLVER, props);
BundleContext context = CoreRuntimePlugin.getInstance().getBundle().getBundleContext();
ServiceReference ca = context.getServiceReference(ConfigurationAdmin.class);
debug("" + ca);
ConfigurationAdmin configAdmin = (ConfigurationAdmin) context.getService(ca);
if ( configAdmin == null ) {
Configuration conf = configAdmin.getConfiguration("(service.pid=org.ops4j.pax.url.mvn)");
assertEquals("59", conf.getProperties().get("org.ops4j.pax.url.mvn.socket.readTimeout"));
assertEquals("119", conf.getProperties().get("org.ops4j.pax.url.mvn.socket.connectionTimeout"));
}
}
@Before
@After
public void cleanup() {
try {
String uri = "mvn:org.talend.libraries/test/6.0.0-SNAPSHOT/jar";
MavenArtifact artifact = MavenUrlHelper.parseMvnUrl(uri);
List<MavenArtifact> searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, false, true);
deleteNexusArtifacts(searchRet,false,true);
uri = "mvn:org.talend.libraries/test/6.0.0/jar";
artifact = MavenUrlHelper.parseMvnUrl(uri);
searchRet = repHandler.search(artifact.getGroupId(), artifact.getArtifactId(), null, true, false);
deleteNexusArtifacts(searchRet,true,true);
} catch ( Exception ex ) {
}
}
private void clearLocalFile(String uri) {
LocalLibraryManager localLibraryManager = new LocalLibraryManager();
String localJarPath = localLibraryManager.getJarPathFromMaven(uri);
debug("localJarPath:" + localJarPath);
// force to delete the file to have a valid test
if (localJarPath != null) {
org.talend.utils.io.FilesUtils.deleteFolder(new File(localJarPath).getParentFile().getParentFile(), true);
}
// file should not exist anymore
assertNull(localLibraryManager.getJarPathFromMaven(uri));
}
private static void createNexusRepository(String repId,String versionPolicy) throws IOException {
if (!"SNAPSHOT".equalsIgnoreCase(versionPolicy) && !"RELEASE".equalsIgnoreCase(versionPolicy)) {
Assert.fail("Repository Version Policy must be SNAPSHOT OR RELEASE, but got " + versionPolicy);
}
String getstmt = customNexusServer.getServer() +
nexusprops.getProperty("rep.list.endpoint").replace("{repid}", repId);
try {
String[] response = RestAPIUtil.doRequest(getstmt, "GET", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
if ( response[0].equals("200")) return;
} catch (Exception ex ) {
//Can not find and will create a new one
debug("Can not find the repository named as " + repId + " and will create a new one :" + ex.getMessage());
}
String createstmt = customNexusServer.getServer() + nexusprops.getProperty("rep.create.endpoint");
String jsonfilepath = FileLocator.toFileURL(Platform.getBundle("org.talend.librariesmanager.test").getEntry("resources/nexus/nexus3_create_rep.json")).getFile();
String data = new String(Files.readAllBytes(new File(jsonfilepath).toPath())).replace("NEWREPNAME", repId).replace("NEWVERSIONPOLICY", versionPolicy.toUpperCase());
try {
String[] response = RestAPIUtil.doRequest(createstmt, "POST", customNexusServer.getUserName(), customNexusServer.getPassword(), data);
if ( response[0].equals("201")) debug("Created the test repository successfully!");
} catch (Exception ex ) {
debug("Exception when create the repository of " + repId + ":" + ex );
}
}
private static void deleteNexusArtifacts( List<MavenArtifact> artifacts,boolean isRelease, boolean skiperror) throws Exception{
if (artifacts == null) return;
for (MavenArtifact artifact : artifacts) {
deleteNexusArtifact(artifact,isRelease,skiperror);
}
}
private static void deleteNexusArtifact(MavenArtifact artifact, boolean fromRelease, boolean skiperror) throws Exception {
try {
StringBuffer searchstmt = new StringBuffer();
searchstmt.append(customNexusServer.getServer())
.append(nexusprops.getProperty("component.search.endpoint"))
.append("?repository=").append( fromRelease ? customNexusServer.getRepositoryId() : customNexusServer.getSnapshotRepId())
.append("&group=").append(artifact.getGroupId())
.append("&name=").append(artifact.getArtifactId());
if ( !artifact.getVersion().endsWith(MavenUrlHelper.VERSION_SNAPSHOT))
searchstmt.append("&version=").append(artifact.getVersion());
String[] resp = RestAPIUtil.doRequest(searchstmt.toString(), "GET", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
if ( !resp[0].equals("200")) return;
JSONObject obj = JSONObject.fromObject(resp[1]);
JSONArray items = obj.getJSONArray("items");
if ( items.size() < 1) return;
JSONObject item = (JSONObject) items.get(0);
String componentid = item.getString("id");
String deltestmt = customNexusServer.getServer() + nexusprops.getProperty("component.delete.endpoint").replace("{id}", componentid);
RestAPIUtil.doRequest(deltestmt, "DELETE", customNexusServer.getUserName(), customNexusServer.getPassword(), null);
} catch (Exception ex) {
if (!skiperror) throw ex;
}
}
private String getSha1(File file) throws IOException {
FileInputStream fis = new FileInputStream(file);
String sha1 = DigestUtils.shaHex(fis);
fis.close();
return sha1;
}
private String getSha1(String file) throws IOException {
return getSha1(new File(file));
}
private static void debug(String message) {
System.out.println("[DEBUG]:" + message);
}
}

View File

@@ -0,0 +1,52 @@
package org.talend.librariesmanager.nexus.utils;
import java.util.Properties;
import org.talend.core.nexus.ArtifactRepositoryBean;
import org.talend.core.nexus.IRepositoryArtifactHandler;
import org.talend.core.nexus.RepositoryArtifactHandlerManager;
public class NexusServerManagerProxy {
private static String NEXUS_USER = "nexus.user";
private static String NEXUS_PASSWORD = "nexus.password";
private static String NEXUS_URL = "nexus.url";
private static String NEXUS_LIB_REPO = "nexus.lib.repo";
private static String NEXUS_LIB_SNAPSHOT_REPO = "nexus.lib.repo.snapshot";
private static String DEFAULT_LIB_REPO = "rep-for-test-releases";
private static String DEFAULT_LIB_SNAPSHOT_REPO = "rep-for-test-snapshots";
private static String NEXUS_LIB_SERVER_TYPE = "nexus.lib.server.type";
private static Properties props;
private static NexusServerManagerProxy manager = null;
public static synchronized NexusServerManagerProxy getInstance() {
if (manager == null) {
manager = new NexusServerManagerProxy();
}
return manager;
}
public ArtifactRepositoryBean getCustomNexusServer() {
String nexus_url = System.getProperty(NEXUS_URL);
String nexus_user = System.getProperty(NEXUS_USER);
String nexus_pass = System.getProperty(NEXUS_PASSWORD);
String repositoryId = System.getProperty(NEXUS_LIB_REPO, DEFAULT_LIB_REPO);
String snapshotRepId = System.getProperty(NEXUS_LIB_SNAPSHOT_REPO, DEFAULT_LIB_SNAPSHOT_REPO);
String serverType = System.getProperty(NEXUS_LIB_SERVER_TYPE, "NEXUS_3");
ArtifactRepositoryBean serverBean = new ArtifactRepositoryBean();
serverBean.setServer(nexus_url);
serverBean.setUserName(nexus_user);
serverBean.setPassword(nexus_pass);
serverBean.setRepositoryId(repositoryId);
serverBean.setSnapshotRepId(snapshotRepId);
serverBean.setType(serverType);
return serverBean;
}
public ArtifactRepositoryBean getDummyNexusServer() {
ArtifactRepositoryBean serverBean = getCustomNexusServer();
serverBean.setServer("http://localhost:0000");
return serverBean;
}
}

View File

@@ -0,0 +1,57 @@
package org.talend.librariesmanager.nexus.utils;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
public class RestAPIUtil {
public static String[] doRequest(String uri, String httpmethod, String user, String password, String data)
throws Exception {
String[] ret = new String[2];
StringBuffer response = new StringBuffer();
URL url = new URL(uri);
String auth = user + ":" + password;
try {
byte[] encodedAuth = Base64.getEncoder().encode((auth.getBytes(StandardCharsets.UTF_8)));
String authHeaderValue = "Basic " + new String(encodedAuth);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Authorization", authHeaderValue);
conn.setRequestProperty("Content-Type", "application/json");
conn.setRequestProperty("Accept", "application/json");
conn.setRequestMethod(httpmethod);
if (data != null) {
conn.setDoOutput(true);
conn.setDoInput(true);
conn.connect();
try (OutputStreamWriter streamWriter = new OutputStreamWriter(conn.getOutputStream());) {
streamWriter.write(data);
streamWriter.flush();
}
}
int responsecode = conn.getResponseCode();
debug(responsecode + " is returned for " + httpmethod + " on the endpoint:" + uri);
ret[0] = Integer.toString(responsecode);
try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
String responseLine = null;
while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim());
}
}
ret[1] = response.toString();
conn.disconnect();
} catch (Exception ex) {
debug(ex.getMessage());
throw ex;
}
return ret;
}
private static void debug(String message) {
System.out.println("[DEBUG]:" + message);
}
}

View File

@@ -0,0 +1,32 @@
package org.talend.librariesmanager.nexus.utils;
import org.junit.Assert;
import org.junit.Test;
public class ShareLibrariesUtilTest {
@Test
public void testGetMavenClassifier() {
String path = "javax/xml/bind/acxb-test/2.2.6/acxb-test-2.2.6-jdk10.dll";
String classifier = ShareLibrariesUtil.getMavenClassifier(path, "acxb-test-2.2.6", "dll");
Assert.assertEquals(classifier, "jdk10");
//
String path1 = "org/talend/libraries/aa/6.0.0-SNAPSHOT/aa-6.0.0-20201027.064528-1.dll";
String classifier1 = ShareLibrariesUtil.getMavenClassifier(path1, "aa-6.0.0-SNAPSHOT", "dll");
Assert.assertNull(classifier1);
String path2 = "org/talend/libraries/ldapjdk/6.0.0/ldapjdk-6.0.0-jdk9.dll";
String classifier2 = ShareLibrariesUtil.getMavenClassifier(path2, "ldapjdk-6.0.0", "dll");
Assert.assertEquals(classifier2, "jdk9");
//
String path3 = "org/talend/libraries/acxb-test-2.2.6-jdk8/6.0.0-SNAPSHOT/acxb-test-2.2.6-jdk8-6.0.0-20201103.062422-1.dll";
String classifier3 = ShareLibrariesUtil.getMavenClassifier(path3, "acxb-test-2.2.6-jdk8-6.0.0-SNAPSHOT", "dll");
Assert.assertNull(classifier3);
String path4 = "org/talend/libraries/acxb-test-2.2.6-jdk8/6.0.0-SNAPSHOT/acxb-test-2.2.6-jdk8-6.0.0-SNAPSHOT-jdk8.dll";
String classifier4 = ShareLibrariesUtil.getMavenClassifier(path4, "acxb-test-2.2.6-jdk8-6.0.0-SNAPSHOT", "dll");
Assert.assertEquals(classifier4, "jdk8");
}
}