Compare commits

...

4 Commits

Author SHA1 Message Date
xwen
9f774b05a9 feat(TUP-18192)Intelligent Auto-Map,leveraging Data Quality algorithms
*preview system that abandoned for now.
*linker is not finished yet

https://jira.talendforge.org/browse/TUP-18192
2017-08-21 14:47:06 +08:00
xwen
8e3b894b40 feat(TUP-18192)Intelligent Auto-Map,leveraging Data Quality algorithms
https://jira.talendforge.org/browse/TUP-18192
2017-08-09 15:32:21 +08:00
xwen
ba09021891 feat(TUP-18192)Intelligent Auto-Map,leveraging Data Quality algorithms
*Levenshtein and Jaccard done

https://jira.talendforge.org/browse/TUP-18192
2017-08-03 11:14:48 +08:00
xwen
54a65f277e feat(TUP-18354)Intelligent Auto-Map,leveraging Data Quality algorithms
*subtask of TUP-18192
*implement Levenshtein and improvement

https://jira.talendforge.org/browse/TUP-18354
2017-08-01 16:11:03 +08:00
19 changed files with 1162 additions and 26 deletions

View File

@@ -113,6 +113,10 @@ public abstract class AbstractUIManager {
// do nothing.
}
public void openAutoMappingDialog() {
// do nothing.
}
public abstract AbstractMapperManager getAbstractMapperManager();
public abstract void refreshBackground(boolean forceRecalculate, boolean firstExecutionAfterTime);

View File

@@ -31,6 +31,8 @@ public interface ITableEntry {
public String getParentName();
public String getTableName();
public IDataMapTable getParent();
public String getExpression();

View File

@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/main/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="output" path="class"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/main/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="output" path="class"/>
</classpath>

View File

@@ -45,3 +45,4 @@ Export-Package: org.talend.designer.mapper,
org.talend.designer.mapper.model.tableentry,
org.talend.designer.mapper.utils,
org.talend.designer.mapper.utils.problems
Import-Package: org.eclipse.jface.preference

View File

@@ -0,0 +1,50 @@
// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.designer.mapper.managers;
/**
* DOC xwen class global comment. Detailled comment
*/
public class MapperAutoMappingManager {
private static MapperAutoMappingManager instance = null;
private MapperManager manager;
/**
* DOC xwen MapperAutoMappingManager constructor comment.
*
* @param manager
*/
public MapperAutoMappingManager(MapperManager manager) {
this.manager = manager;
initModelSettings();
}
public static synchronized MapperAutoMappingManager getInstance(MapperManager manager) {
if (instance == null) {
instance = new MapperAutoMappingManager(manager);
}
return instance;
}
/**
* DOC xwen Comment method "initModelSettings".
*/
private void initModelSettings() {
}
}

View File

@@ -1035,9 +1035,9 @@ public class MapperManager extends AbstractMapperManager {
/**
* DOC amaumont Comment method "mapAutomaticallly".
*/
public void mapAutomaticallly() {
public void mapAutomaticallly(int levenshteinWeight, int jaccardWeight) {
AutoMapper autoMapper = new AutoMapper(this);
autoMapper.map();
autoMapper.map(levenshteinWeight, jaccardWeight);
}
/**

View File

@@ -117,6 +117,7 @@ import org.talend.designer.mapper.model.tableentry.TableEntryLocation;
import org.talend.designer.mapper.model.tableentry.VarTableEntry;
import org.talend.designer.mapper.ui.MapperUI;
import org.talend.designer.mapper.ui.commands.DataMapTableViewSelectedCommand;
import org.talend.designer.mapper.ui.dialog.AutoMappingDialog;
import org.talend.designer.mapper.ui.dialog.MapReducePropertySetDialog;
import org.talend.designer.mapper.ui.dialog.OutputAddDialog;
import org.talend.designer.mapper.ui.dialog.PropertySetDialog;
@@ -2017,6 +2018,16 @@ public class UIManager extends AbstractUIManager {
}
}
/*
* (non-Javadoc)
*
* @see org.talend.designer.abstractmap.managers.AbstractUIManager#openAutoMapDialog()
*/
@Override
public void openAutoMappingDialog() {
new AutoMappingDialog(getShell(), mapperManager).open();
}
public boolean isTableViewMoveable(Zone zone, boolean moveUp) {
if (zone == Zone.INPUTS) {
if (currentSelectedInputTableView == null) {

View File

@@ -30,6 +30,7 @@ public class MapperSettingModel implements Cloneable {
private String rowBufferSize;
/**
* @return the MapReduce lookup tables should be precalculated and distributed across nodes in the cluster.
*/

View File

@@ -26,6 +26,13 @@ public class AbstractInOutTableEntry extends DataMapTableEntry implements IColum
private IMetadataColumn metadataColumn;
private String parentName;
public AbstractInOutTableEntry(IMetadataColumn metadataColumn) {
this.metadataColumn = metadataColumn;
}
public AbstractInOutTableEntry(IDataMapTable abstractDataMapTable, IMetadataColumn metadataColumn, String expression) {
super(abstractDataMapTable, expression);
this.metadataColumn = metadataColumn;
@@ -70,4 +77,24 @@ public class AbstractInOutTableEntry extends DataMapTableEntry implements IColum
return true;
}
/**
* Getter for parentName.
*
* @return the parentName
*/
/**
* Sets the parentName.
*
* @param parentName the parentName to set
*/
public void setParentName(String parentName) {
this.parentName = parentName;
}
public String getTableName() {
return this.parentName;
}
}

View File

@@ -39,6 +39,10 @@ public abstract class DataMapTableEntry implements ITableEntry {
private List<Problem> problems;
public DataMapTableEntry() {
}
public DataMapTableEntry(IDataMapTable abstractDataMapTable, String expression) {
super();
this.parent = abstractDataMapTable;
@@ -97,6 +101,10 @@ public abstract class DataMapTableEntry implements ITableEntry {
}
}
public String getTableName() {
return null;
}
public List<Problem> getProblems() {
return this.problems;
}

View File

@@ -25,6 +25,7 @@ public class InputColumnTableEntry extends AbstractInOutTableEntry {
private String operator;
public InputColumnTableEntry(IDataMapTable abstractDataMapTable, IMetadataColumn metadataColumn, String expression) {
super(abstractDataMapTable, metadataColumn, expression);
}
@@ -33,6 +34,13 @@ public class InputColumnTableEntry extends AbstractInOutTableEntry {
super(abstractDataMapTable, metadataColumn);
}
/**
* DOC xwen InputColumnTableEntry constructor comment.
*/
public InputColumnTableEntry(IMetadataColumn metadataColumn) {
super(metadataColumn);
}
/**
* Getter for operator.
*
@@ -51,4 +59,6 @@ public class InputColumnTableEntry extends AbstractInOutTableEntry {
this.operator = operator;
}
}

View File

@@ -31,4 +31,11 @@ public class OutputColumnTableEntry extends AbstractInOutTableEntry {
super(abstractDataMapTable, metadataColumn);
}
/**
* DOC xwen OutputColumnTableEntry constructor comment.
*/
public OutputColumnTableEntry(IMetadataColumn metadataColumn) {
super(metadataColumn);
}
}

View File

@@ -14,6 +14,7 @@ package org.talend.designer.mapper.ui.automap;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import org.talend.designer.abstractmap.model.tableentry.IColumnEntry;
import org.talend.designer.mapper.language.ILanguage;
@@ -22,6 +23,9 @@ import org.talend.designer.mapper.managers.MapperManager;
import org.talend.designer.mapper.model.table.InputTable;
import org.talend.designer.mapper.model.table.OutputTable;
import org.talend.designer.mapper.ui.visualmap.table.DataMapTableView;
import org.talend.utils.string.Jaccard;
import org.talend.utils.string.Levenshtein;
/**
* DOC amaumont class global comment. Detailled comment <br/>
@@ -45,7 +49,10 @@ public class AutoMapper {
/**
* DOC amaumont Comment method "map".
*/
public void map() {
public void map(int levenshteinWeight, int jaccardWeight) {
int paramL = levenshteinWeight;
int paramJ = jaccardWeight;
List<InputTable> inputTables = mapperManager.getInputTables();
List<OutputTable> outputTables = mapperManager.getOutputTables();
@@ -61,30 +68,41 @@ public class AutoMapper {
for (OutputTable outputTable : outputTables) {
List<IColumnEntry> outputEntries = outputTable.getColumnEntries();
boolean mapFound = false;
for (IColumnEntry outputEntry : outputEntries) {
mapFound = false;
if (mapperManager.checkEntryHasEmptyExpression(outputEntry)) {
String outputColumnName = outputEntry.getName();
String outputColumnName = outputEntry.getName().toLowerCase();
String jaccardOutput = Jaccard.tokenize(outputEntry.getName());
HashMap<IColumnEntry, Double> finalMap = new HashMap<IColumnEntry, Double>();
for (InputTable inputTable : inputTables) {
List<IColumnEntry> inputColumnEntries = inputTable.getColumnEntries();
for (IColumnEntry inputEntry : inputColumnEntries) {
if (inputEntry.getName().equalsIgnoreCase(outputColumnName)) {
outputEntry.setExpression(currentLanguage.getLocation(inputTable.getName(), inputEntry
.getName()));
mapFound = true;
break;
}
}
if (mapFound) {
break;
}
// Levenshtein
String inputStr = inputEntry.getName().toLowerCase();
double LevenshteinScore = Levenshtein.getLevenshteinScore(inputStr, outputColumnName);
// Jaccard
String jaccardIutput = Jaccard.tokenize(inputEntry.getName());
double JaccardScore = Jaccard.JaccardCompare(jaccardIutput, jaccardOutput);
double finalScore = LevenshteinScore * paramL + JaccardScore * paramJ;
finalMap.put(inputEntry, finalScore);
inputEntry.getParent();
}
}
IColumnEntry bestEntry = getMaxStr(finalMap);
if (bestEntry == null) {
continue;
}
if (finalMap.get(bestEntry) < 30) {
continue;
} else {
outputEntry
.setExpression(currentLanguage.getLocation(bestEntry.getParent().getName(), bestEntry.getName()));
}
}
@@ -106,4 +124,22 @@ public class AutoMapper {
}
public static IColumnEntry getMaxStr(HashMap<IColumnEntry, Double> map) {
Double max = 0.0;
IColumnEntry result = null;
for (Entry<IColumnEntry, Double> entry : map.entrySet()) {
if (entry.getValue() > max) {
result = entry.getKey();
if (result != null)
max = entry.getValue();
}
}
return result;
}
}

View File

@@ -0,0 +1,650 @@
// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.designer.mapper.ui.dialog;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import org.eclipse.jface.dialogs.Dialog;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.viewers.TableViewer;
import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.SashForm;
import org.eclipse.swt.custom.ScrolledComposite;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.layout.GridData;
import org.eclipse.swt.layout.GridLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Combo;
import org.eclipse.swt.widgets.Composite;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Group;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Slider;
import org.eclipse.swt.widgets.Table;
import org.eclipse.swt.widgets.TableColumn;
import org.talend.core.model.metadata.IMetadataColumn;
import org.talend.core.model.metadata.MetadataColumn;
import org.talend.core.prefs.ITalendCorePrefConstants;
import org.talend.core.ui.CoreUIPlugin;
import org.talend.designer.abstractmap.model.table.IDataMapTable;
import org.talend.designer.abstractmap.model.tableentry.IColumnEntry;
import org.talend.designer.mapper.language.ILanguage;
import org.talend.designer.mapper.language.LanguageProvider;
import org.talend.designer.mapper.managers.MapperManager;
import org.talend.designer.mapper.model.MapperModel;
import org.talend.designer.mapper.model.table.AbstractDataMapTable;
import org.talend.designer.mapper.model.table.InputTable;
import org.talend.designer.mapper.model.table.OutputTable;
import org.talend.designer.mapper.model.tableentry.InputColumnTableEntry;
import org.talend.designer.mapper.model.tableentry.OutputColumnTableEntry;
import org.talend.designer.mapper.ui.visualmap.zone.scrollable.InputTablesZoneView;
import org.talend.designer.mapper.ui.visualmap.zone.scrollable.OutputTablesZoneView;
import org.talend.utils.string.Jaccard;
import org.talend.utils.string.Levenshtein;
/**
* DOC xwen class global comment. Detailled comment
*/
public class AutoMappingDialog extends Dialog {
private IPreferenceStore weightStore;
private MapperManager mapperManager;
private MapperModel mapperModel;
private Slider levenshteinSlider;
private Slider jaccardSlider;
private int levenshteinWeight = 0;
private Integer jaccardWeight = 0;
private Label levenshteinWeightLabel;
private Label jaccardWeightLabel;
private Group autoMapGroup;
private Button previewButton;
private Button resetButton;
private SashForm datasFlowViewSashForm;
private Composite inputsZone;
private Composite outputZone;
private Composite container;
private SashForm mainSash;
private ScrolledComposite sc2;
private InputTablesZoneView inputTablesZoneView;
private OutputTablesZoneView outputTablesZoneView;
private List<InputTable> inputTables;
private List<List<IColumnEntry>> cacheInputTables = new ArrayList<List<IColumnEntry>>();
private List<List<IColumnEntry>> cacheOutputTables = new ArrayList<List<IColumnEntry>>();
private List<List<IColumnEntry>> resetOutTables = new ArrayList<List<IColumnEntry>>();
private List<List<IColumnEntry>> resetInTables = new ArrayList<List<IColumnEntry>>();
private List<IColumnEntry> currentInputList = new ArrayList<IColumnEntry>();
private List<OutputTable> outputTables;
private List<IColumnEntry> currentOutputList = new ArrayList<IColumnEntry>();
private static final boolean SHOW_BORDERS = false;
private TableViewer tableViewerOut;
private TableViewer tableViewerIn;
private Combo inCombo;
private Combo outCombo;
/**
* DOC xwen AutoMappingDialog constructor comment.
*
* @param parentShell
*/
public AutoMappingDialog(Shell parentShell, MapperManager mapperManager) {
super(parentShell);
this.mapperManager = mapperManager;
}
@Override
protected int getShellStyle() {
return super.getShellStyle() | SWT.RESIZE | SWT.MAX;
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.dialogs.Dialog#createDialogArea(org.eclipse.swt.widgets.Composite)
*/
@Override
protected Control createDialogArea(Composite parent) {
container = new Composite(parent, SWT.RESIZE);
// container = (Composite) super.createDialogArea(parent);
final GridLayout gridLayout = new GridLayout(1, false);
gridLayout.marginLeft = 10;
gridLayout.marginTop = 10;
gridLayout.marginHeight = 10;
container.setLayout(gridLayout);
autoMapGroup = new Group(container, SWT.NONE);
autoMapGroup.setLayoutData(new GridData(SWT.FILL, SWT.FILL, true, false));
autoMapGroup.setText("Auto Map");
GridLayout AutogridLayout = new GridLayout(3, false);
AutogridLayout.horizontalSpacing = 10;
AutogridLayout.marginRight = 100;
autoMapGroup.setLayout(AutogridLayout);
Label levenshteinLabel = new Label(autoMapGroup, SWT.NONE);
levenshteinLabel.setText("Levenshtein");
levenshteinSlider = new Slider(autoMapGroup, SWT.HORIZONTAL);
levenshteinSlider.setSize(200, 25);
levenshteinSlider.setMaximum(101);
levenshteinSlider.setMinimum(0);
levenshteinSlider.setThumb(1);
levenshteinWeightLabel = new Label(autoMapGroup, SWT.NONE);
Label jaccardLabel = new Label(autoMapGroup, SWT.NONE);
jaccardLabel.setText("Jaccard");
jaccardSlider = new Slider(autoMapGroup, SWT.HORIZONTAL);
jaccardSlider.setSize(200, 25);
jaccardSlider.setMaximum(101);
jaccardSlider.setMinimum(0);
jaccardSlider.setThumb(1);
jaccardWeightLabel = new Label(autoMapGroup, SWT.NONE);
Composite buttonComposite = new Composite(container, SWT.NONE);
FormLayout layout = new FormLayout();
FormData data = new FormData();
data.left = new FormAttachment(90, 20);
previewButton = new Button(buttonComposite, SWT.NONE);
previewButton.setText("Preview");
resetButton = new Button(buttonComposite, SWT.NONE);
resetButton.setText(" Reset ");
resetButton.setLayoutData(data);
buttonComposite.setLayout(layout);
mainSash = new SashForm(container, SWT.SMOOTH | SWT.HORIZONTAL | SWT.BORDER);
init();
initTables();
addListener();
return container;
}
/**
* DOC xwen Comment method "initTables".
*/
private void initTables() {
createInputZone();
createOutputZone();
}
/**
* DOC xwen Comment method "createOutputZone".
*/
private void createOutputZone() {
outputZone = new Composite(mainSash, SWT.NONE);
GridLayout grid = new GridLayout(1, true);
outputZone.setLayout(grid);
Composite comboComp = new Composite(outputZone, SWT.NONE);
GridLayout layout = new GridLayout(2, true);
Label outLabel = new Label(comboComp, SWT.NONE);
outLabel.setText("Output Tables");
comboComp.setLayout(layout);
outCombo = new Combo(comboComp, SWT.READ_ONLY);
sc2 = new ScrolledComposite(outputZone, SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL);
GridData sc2GridData = new GridData(GridData.FILL_BOTH);
sc2.setLayoutData(sc2GridData);
sc2.setBackgroundMode(SWT.INHERIT_DEFAULT);
sc2.setLayout(new GridLayout(1, false));
outputTablesZoneView = new OutputTablesZoneView(sc2, getBorder(), mapperManager);
outputTablesZoneView.setBackgroundMode(SWT.INHERIT_DEFAULT);
sc2.setContent(outputTablesZoneView);
GridLayout grid1 = new GridLayout();
outputZone.setLayout(grid1);
outputTablesZoneView.setLayout(grid1);
sc2.setExpandHorizontal(true);
outputTablesZoneView.setSize(200, 200);
sc2.layout();
tableViewerOut = new TableViewer(outputTablesZoneView, SWT.BORDER | SWT.FULL_SELECTION);
Table table = tableViewerOut.getTable();
table.setHeaderVisible(true);
table.setLinesVisible(true);
TableColumn col1 = new TableColumn(table, SWT.NONE);
col1.setText("Expression");
col1.setWidth(150);
TableColumn col2 = new TableColumn(table, SWT.NONE);
col2.setText("Column");
col2.setWidth(150);
TableViewerContentProvider provider = new TableViewerContentProvider();
OutTableViewerLabelProvider labelProvider = new OutTableViewerLabelProvider();
tableViewerOut.setContentProvider(provider);
tableViewerOut.setLabelProvider(labelProvider);
// Object data = null;
OutputTable firstTable = null;
if (!outputTables.isEmpty()) {
firstTable = outputTables.get(0);
}
if (firstTable != null) {
List<IColumnEntry> outputColumnEntries = firstTable.getColumnEntries();
// data = outputColumnEntries;
for (IColumnEntry outputEntry : outputColumnEntries) {
IMetadataColumn metadataColumn = new MetadataColumn();
OutputColumnTableEntry entry = new OutputColumnTableEntry(metadataColumn);
entry.setName(outputEntry.getName());
entry.setExpression(outputEntry.getExpression());
entry.setParentName(outputEntry.getParent().getName());
currentOutputList.add(entry);
}
}
tableViewerOut.setUseHashlookup(true);
// cache all output tables
initOutputTables();
outCombo.select(0);
tableViewerOut.setInput(cacheOutputTables.get(0));
outputTablesZoneView.setSize(outputTablesZoneView.computeSize(SWT.DEFAULT, SWT.DEFAULT));
}
/**
* DOC xwen Comment method "createInputZone".
*/
private void createInputZone() {
inputsZone = new Composite(mainSash, SWT.NONE);
GridLayout grid = new GridLayout(1, true);
inputsZone.setLayout(grid);
Composite comboComp = new Composite(inputsZone, SWT.NONE);
GridLayout layout = new GridLayout(2, true);
Label inLabel = new Label(comboComp, SWT.NONE);
inLabel.setText("Intput Tables");
comboComp.setLayout(layout);
inCombo = new Combo(comboComp, SWT.READ_ONLY);
ScrolledComposite sc1 = new ScrolledComposite(inputsZone, SWT.BORDER | SWT.H_SCROLL | SWT.V_SCROLL);
GridData sc1GridData = new GridData(GridData.FILL_BOTH);
sc1.setLayoutData(sc1GridData);
// sc1.setLayout(new GridLayout(1, false));
sc1.setBackgroundMode(SWT.INHERIT_DEFAULT);
inputTablesZoneView = new InputTablesZoneView(sc1, getBorder(), mapperManager);
inputTablesZoneView.setBackgroundMode(SWT.INHERIT_DEFAULT);
inputTablesZoneView.setSize(100, 100);
sc1.setExpandHorizontal(true);
sc1.setExpandVertical(true);
sc1.setMinWidth(100);
sc1.setMinHeight(100);
sc1.setBounds(50, 50, 300, 300);
sc1.setContent(inputTablesZoneView);
// sc1.setSize(100, 100);
tableViewerIn = new TableViewer(inputTablesZoneView, SWT.BORDER | SWT.FULL_SELECTION);
Table table = tableViewerIn.getTable();
table.setHeaderVisible(true);
table.setLinesVisible(true);
TableColumn col = new TableColumn(table, SWT.NONE);
col.setText("Column");
col.setWidth(150);
TableViewerContentProvider provider = new TableViewerContentProvider();
InTableViewerLabelProvider labelProvider = new InTableViewerLabelProvider();
tableViewerIn.setContentProvider(provider);
tableViewerIn.setLabelProvider(labelProvider);
tableViewerIn.setUseHashlookup(true);
//cache all input tables
initInputTables();
inCombo.select(0);
tableViewerIn.setInput(cacheInputTables.get(0));
inputTablesZoneView.setSize(inputTablesZoneView.computeSize(SWT.DEFAULT, SWT.DEFAULT));
}
private void initInputTables() {
if (cacheInputTables.isEmpty()) {
for (InputTable inputTable : inputTables) {
List<IColumnEntry> inputColumnEntries = inputTable.getColumnEntries();
List<IColumnEntry> cacheInputTable = new ArrayList<IColumnEntry>();
for (IColumnEntry inputEntry : inputColumnEntries) {
IMetadataColumn metadataColumn = new MetadataColumn();
InputColumnTableEntry entry = new InputColumnTableEntry(metadataColumn);
entry.setName(inputEntry.getName());
entry.setParentName(inputEntry.getParent().getName());
cacheInputTable.add(entry);// put entry into table
}
cacheInputTables.add(cacheInputTable);
inCombo.add(inputTable.getName());
}
} else {
cacheInputTables = new ArrayList<List<IColumnEntry>>();
for (InputTable inputTable : inputTables) {
List<IColumnEntry> inputColumnEntries = inputTable.getColumnEntries();
List<IColumnEntry> cacheInputTable = new ArrayList<IColumnEntry>();
for (IColumnEntry inputEntry : inputColumnEntries) {
IMetadataColumn metadataColumn = new MetadataColumn();
InputColumnTableEntry entry = new InputColumnTableEntry(metadataColumn);
entry.setName(inputEntry.getName());
entry.setParentName(inputEntry.getParent().getName());
cacheInputTable.add(entry);// put entry into table
}
cacheInputTables.add(cacheInputTable);
}
}
}
private void initOutputTables() {
if (cacheOutputTables.isEmpty()) {
for (OutputTable outputTable : outputTables) {
List<IColumnEntry> outputColumnEntries = outputTable.getColumnEntries();
List<IColumnEntry> cacheOutputTable = new ArrayList<IColumnEntry>();
for (IColumnEntry outputEntry : outputColumnEntries) {
IMetadataColumn metadataColumn = new MetadataColumn();
OutputColumnTableEntry entry = new OutputColumnTableEntry(metadataColumn);
entry.setName(outputEntry.getName());
entry.setExpression(outputEntry.getExpression());
entry.setParentName(outputEntry.getParent().getName());
cacheOutputTable.add(entry);// put entry into table
}
cacheOutputTables.add(cacheOutputTable);
outCombo.add(outputTable.getName());
}
} else {
cacheOutputTables = new ArrayList<List<IColumnEntry>>();
for (OutputTable outputTable : outputTables) {
List<IColumnEntry> outputColumnEntries = outputTable.getColumnEntries();
List<IColumnEntry> cacheOutputTable = new ArrayList<IColumnEntry>();
for (IColumnEntry outputEntry : outputColumnEntries) {
IMetadataColumn metadataColumn = new MetadataColumn();
OutputColumnTableEntry entry = new OutputColumnTableEntry(metadataColumn);
entry.setName(outputEntry.getName());
entry.setExpression(outputEntry.getExpression());
entry.setParentName(outputEntry.getParent().getName());
cacheOutputTable.add(entry);// put entry into table
}
cacheOutputTables.add(cacheOutputTable);
}
}
}
/**
* DOC xwen Comment method "getMinimizedButtonState".
*
* @param tables
* @return
*/
private Boolean getMinimizedButtonState(List<? extends AbstractDataMapTable> tables) {
boolean allTablesAreMinimized = true;
boolean allTablesAreNotMinimized = true;
for (IDataMapTable table : tables) {
if (table.isMinimized()) {
allTablesAreNotMinimized = false;
} else {
allTablesAreMinimized = false;
}
}
if (allTablesAreMinimized) {
return new Boolean(true);
} else if (allTablesAreNotMinimized) {
return new Boolean(false);
}
return new Boolean(false);
}
/**
* DOC xwen Comment method "init".
*/
private void init() {
weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
levenshteinWeightLabel.setText(String.valueOf(weightStore.getInt(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT)));
levenshteinSlider.setSelection(weightStore.getInt(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT));
jaccardWeightLabel.setText(String.valueOf(weightStore.getInt(ITalendCorePrefConstants.JACCARD_WEIGHT)));
jaccardSlider.setSelection(weightStore.getInt(ITalendCorePrefConstants.JACCARD_WEIGHT));
this.mapperModel = new MapperModel(mapperManager.getInputTables(), mapperManager.getOutputTables(),
mapperManager.getVarsTables());
inputTables = mapperManager.getInputTables();
outputTables = mapperManager.getOutputTables();
}
private void addListener() {
levenshteinSlider.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
updateAutoMap();
}
});
jaccardSlider.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
updateAutoMap();
}
});
previewButton.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
// IPreferenceStore weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
int levenshteinWeight = levenshteinSlider.getSelection();
int jaccardWeight = jaccardSlider.getSelection();
map(levenshteinWeight, jaccardWeight);
}
});
resetButton.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
initInputTables();
initOutputTables();
tableViewerIn.setInput(cacheInputTables.get(inCombo.getSelectionIndex()));
tableViewerOut.setInput(cacheOutputTables.get(outCombo.getSelectionIndex()));
}
});
inCombo.addSelectionListener(new SelectionAdapter() {
public void widgetSelected(SelectionEvent e) {
tableViewerIn.setInput(cacheInputTables.get(inCombo.getSelectionIndex()));
};
});
outCombo.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
tableViewerOut.setInput(cacheOutputTables.get(outCombo.getSelectionIndex()));
}
});
}
protected void map(int levenshteinWeight, int jaccardWeight) {
int paramL = levenshteinWeight;
int paramJ = jaccardWeight;
ILanguage currentLanguage = LanguageProvider.getCurrentLanguage();
for (List<IColumnEntry> OutputList : cacheOutputTables) {
for (IColumnEntry outputEntry : OutputList) {
if (outputEntry.getExpression() == null || outputEntry.getExpression().trim().length() == 0) {
String outputColumnName = outputEntry.getName().toLowerCase();
String jaccardOutput = Jaccard.tokenize(outputEntry.getName());
HashMap<IColumnEntry, Double> finalMap = new HashMap<IColumnEntry, Double>();
for (List<IColumnEntry> InputList : cacheInputTables) {
for (IColumnEntry inputEntry : InputList) {
String inputStr = inputEntry.getName().toLowerCase();
double LevenshteinScore = Levenshtein.getLevenshteinScore(inputStr, outputColumnName);
// Jaccard
String jaccardIutput = Jaccard.tokenize(inputEntry.getName());
double JaccardScore = Jaccard.JaccardCompare(jaccardIutput, jaccardOutput);
double finalScore = LevenshteinScore * paramL + JaccardScore * paramJ;
finalMap.put(inputEntry, finalScore);
}
}
IColumnEntry bestEntry = getMaxStr(finalMap);
if (bestEntry == null) {
continue;
}
if (finalMap.get(bestEntry) < 30) {
continue;
} else {
outputEntry.setExpression(currentLanguage.getLocation(bestEntry.getTableName(), bestEntry.getName()));
}
}
}
}
int index = outCombo.getSelectionIndex();
tableViewerOut.setInput(cacheOutputTables.get(index));
}
public static IColumnEntry getMaxStr(HashMap<IColumnEntry, Double> map) {
Double max = 0.0;
IColumnEntry result = null;
for (Entry<IColumnEntry, Double> entry : map.entrySet()) {
if (entry.getValue() > max) {
result = entry.getKey();
if (result != null)
max = entry.getValue();
}
}
return result;
}
private void updateAutoMap() {
levenshteinWeight = levenshteinSlider.getSelection();
jaccardWeight = jaccardSlider.getSelection();
levenshteinWeightLabel.setText(String.valueOf(levenshteinWeight));
jaccardWeightLabel.setText(String.valueOf(jaccardWeight));
autoMapGroup.layout();
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.dialogs.Dialog#okPressed()
*/
@Override
protected void okPressed() {
IPreferenceStore weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
levenshteinWeight = levenshteinSlider.getSelection();
jaccardWeight = jaccardSlider.getSelection();
weightStore.setValue(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT, levenshteinWeight);
weightStore.setValue(ITalendCorePrefConstants.JACCARD_WEIGHT, jaccardWeight);
// List<DataMapTableView> dataMapTableViews = mapperManager.getUiManager().getInputsTablesView();
// for (DataMapTableView dataMapTableView : dataMapTableViews) {
// dataMapTableView.setParent(oldParent);
// }
super.okPressed();
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.dialogs.Dialog#cancelPressed()
*/
@Override
protected void cancelPressed() {
// List<DataMapTableView> dataMapTableViews = mapperManager.getUiManager().getInputsTablesView();
// for (DataMapTableView dataMapTableView : dataMapTableViews) {
// dataMapTableView.setParent(oldParent);
// }
super.cancelPressed();
}
private int getBorder() {
return SHOW_BORDERS ? SWT.BORDER : SWT.NONE;
}
}

View File

@@ -0,0 +1,88 @@
// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.designer.mapper.ui.dialog;
import org.eclipse.jface.viewers.ILabelProviderListener;
import org.eclipse.jface.viewers.ITableLabelProvider;
import org.eclipse.swt.graphics.Image;
import org.talend.designer.abstractmap.model.tableentry.IColumnEntry;
import org.talend.designer.mapper.model.tableentry.InputColumnTableEntry;
/**
* DOC xwen class global comment. Detailled comment
*/
public class InTableViewerLabelProvider implements ITableLabelProvider {
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#addListener(org.eclipse.jface.viewers.ILabelProviderListener)
*/
public void addListener(ILabelProviderListener listener) {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#dispose()
*/
public void dispose() {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#isLabelProperty(java.lang.Object, java.lang.String)
*/
public boolean isLabelProperty(Object element, String property) {
// TODO Auto-generated method stub
return false;
}
/*
* (non-Javadoc)
*
* @see
* org.eclipse.jface.viewers.IBaseLabelProvider#removeListener(org.eclipse.jface.viewers.ILabelProviderListener)
*/
public void removeListener(ILabelProviderListener listener) {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.ITableLabelProvider#getColumnImage(java.lang.Object, int)
*/
public Image getColumnImage(Object element, int columnIndex) {
// TODO Auto-generated method stub
return null;
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.ITableLabelProvider#getColumnText(java.lang.Object, int)
*/
public String getColumnText(Object element, int columnIndex) {
IColumnEntry entry = (InputColumnTableEntry) element;
String columnName = entry.getName();
return columnName;
}
}

View File

@@ -0,0 +1,93 @@
// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.designer.mapper.ui.dialog;
import org.eclipse.jface.viewers.ILabelProviderListener;
import org.eclipse.jface.viewers.ITableLabelProvider;
import org.eclipse.swt.graphics.Image;
import org.talend.designer.mapper.model.tableentry.OutputColumnTableEntry;
/**
* DOC xwen class global comment. Detailled comment
*/
public class OutTableViewerLabelProvider implements ITableLabelProvider {
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#addListener(org.eclipse.jface.viewers.ILabelProviderListener)
*/
public void addListener(ILabelProviderListener listener) {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#dispose()
*/
public void dispose() {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IBaseLabelProvider#isLabelProperty(java.lang.Object, java.lang.String)
*/
public boolean isLabelProperty(Object element, String property) {
// TODO Auto-generated method stub
return false;
}
/*
* (non-Javadoc)
*
* @see
* org.eclipse.jface.viewers.IBaseLabelProvider#removeListener(org.eclipse.jface.viewers.ILabelProviderListener)
*/
public void removeListener(ILabelProviderListener listener) {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.ITableLabelProvider#getColumnImage(java.lang.Object, int)
*/
public Image getColumnImage(Object element, int columnIndex) {
// TODO Auto-generated method stub
return null;
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.ITableLabelProvider#getColumnText(java.lang.Object, int)
*/
public String getColumnText(Object element, int columnIndex) {
OutputColumnTableEntry entry = (OutputColumnTableEntry) element;
String columnName = entry.getName();
if (columnIndex == 0) {
return entry.getExpression();
}
if (columnIndex == 1) {
return columnName;
}
return null;
}
}

View File

@@ -15,6 +15,7 @@ package org.talend.designer.mapper.ui.dialog;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jface.dialogs.Dialog;
import org.eclipse.jface.dialogs.IDialogConstants;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.swt.SWT;
import org.eclipse.swt.events.ModifyEvent;
import org.eclipse.swt.events.ModifyListener;
@@ -31,11 +32,14 @@ import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Group;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.Slider;
import org.talend.commons.ui.swt.formtools.LabelledDirectoryField;
import org.talend.commons.ui.swt.formtools.LabelledText;
import org.talend.core.model.components.ComponentCategory;
import org.talend.core.model.components.IComponent;
import org.talend.core.model.utils.ContextParameterUtils;
import org.talend.core.prefs.ITalendCorePrefConstants;
import org.talend.core.ui.CoreUIPlugin;
import org.talend.core.ui.component.ComponentsFactoryProvider;
import org.talend.core.utils.TalendQuoteUtils;
import org.talend.designer.mapper.i18n.Messages;
@@ -63,10 +67,25 @@ public class PropertySetDialog extends Dialog {
private LabelledText sizeField;
private Slider levenshteinSlider;
private Slider jaccardSlider;
private int levenshteinWeight = 0;
private Integer jaccardWeight = 0;
private Label levenshteinWeightLabel;
private Label jaccardWeightLabel;
public static final String QUOTATION_MARK = "\""; //$NON-NLS-1$
private Group autoMapGroup;
private final Color color = new Color(Display.getDefault(), 238, 238, 0);
/**
* Create the dialog
*
@@ -119,6 +138,33 @@ public class PropertySetDialog extends Dialog {
label.setText("*");
label.setToolTipText("Required filed.");
autoMapGroup = new Group(container, SWT.NONE);
autoMapGroup.setLayoutData(new GridData(SWT.FILL, SWT.FILL, true, false));
autoMapGroup.setText("Auto Map");
GridLayout AutogridLayout = new GridLayout(3, false);
AutogridLayout.horizontalSpacing = 10;
AutogridLayout.marginRight = 100;
autoMapGroup.setLayout(AutogridLayout);
Label levenshteinLabel = new Label(autoMapGroup, SWT.NONE);
levenshteinLabel.setText("Levenshtein");
levenshteinSlider = new Slider(autoMapGroup, SWT.HORIZONTAL);
levenshteinSlider.setSize(200, 25);
levenshteinSlider.setMaximum(101);
levenshteinSlider.setMinimum(0);
levenshteinSlider.setThumb(1);
levenshteinWeightLabel = new Label(autoMapGroup, SWT.NONE);
Label jaccardLabel = new Label(autoMapGroup, SWT.NONE);
jaccardLabel.setText("Jaccard");
jaccardSlider = new Slider(autoMapGroup, SWT.HORIZONTAL);
jaccardSlider.setSize(200, 25);
jaccardSlider.setMaximum(101);
jaccardSlider.setMinimum(0);
jaccardSlider.setThumb(1);
jaccardWeightLabel = new Label(autoMapGroup, SWT.NONE);
init();
addListener();
updateStatus();
@@ -133,6 +179,12 @@ public class PropertySetDialog extends Dialog {
enableAutoConvertTypeBtn.setSelection(currnentModel.isEnableAutoConvertType());
directoryField.setText(StringUtils.trimToEmpty(currnentModel.getTempDataDir()));
sizeField.setText(StringUtils.trimToEmpty(currnentModel.getRowBufferSize()));
IPreferenceStore weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
levenshteinWeightLabel.setText(String.valueOf(weightStore.getInt(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT)));
levenshteinSlider.setSelection(weightStore.getInt(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT));
jaccardWeightLabel.setText(String.valueOf(weightStore.getInt(ITalendCorePrefConstants.JACCARD_WEIGHT)));
jaccardSlider.setSelection(weightStore.getInt(ITalendCorePrefConstants.JACCARD_WEIGHT));
}
private void addListener() {
@@ -168,6 +220,22 @@ public class PropertySetDialog extends Dialog {
}
});
levenshteinSlider.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
updateAutoMap();
}
});
jaccardSlider.addSelectionListener(new SelectionAdapter() {
@Override
public void widgetSelected(SelectionEvent e) {
updateAutoMap();
}
});
}
private void updateStatus() {
@@ -207,6 +275,14 @@ public class PropertySetDialog extends Dialog {
}
}
private void updateAutoMap() {
levenshteinWeight = levenshteinSlider.getSelection();
jaccardWeight = jaccardSlider.getSelection();
levenshteinWeightLabel.setText(String.valueOf(levenshteinWeight));
jaccardWeightLabel.setText(String.valueOf(jaccardWeight));
autoMapGroup.layout();
}
/**
* Create contents of the button bar
*
@@ -256,6 +332,12 @@ public class PropertySetDialog extends Dialog {
mapperManager.addRejectOutput();
}
}
IPreferenceStore weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
levenshteinWeight = levenshteinSlider.getSelection();
jaccardWeight = jaccardSlider.getSelection();
weightStore.setValue(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT, levenshteinWeight);
weightStore.setValue(ITalendCorePrefConstants.JACCARD_WEIGHT, jaccardWeight);
super.okPressed();
}

View File

@@ -0,0 +1,60 @@
// ============================================================================
//
// Copyright (C) 2006-2016 Talend Inc. - www.talend.com
//
// This source code is available under agreement available at
// %InstallDIR%\features\org.talend.rcp.branding.%PRODUCTNAME%\%PRODUCTNAME%license.txt
//
// You should have received a copy of the agreement
// along with this program; if not, write to Talend SA
// 9 rue Pages 92150 Suresnes, France
//
// ============================================================================
package org.talend.designer.mapper.ui.dialog;
import java.util.List;
import org.eclipse.jface.viewers.IStructuredContentProvider;
import org.eclipse.jface.viewers.Viewer;
/**
* DOC xwen class global comment. Detailled comment
*/
public class TableViewerContentProvider implements IStructuredContentProvider {
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IContentProvider#dispose()
*/
public void dispose() {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IContentProvider#inputChanged(org.eclipse.jface.viewers.Viewer, java.lang.Object,
* java.lang.Object)
*/
public void inputChanged(Viewer viewer, Object oldInput, Object newInput) {
// TODO Auto-generated method stub
}
/*
* (non-Javadoc)
*
* @see org.eclipse.jface.viewers.IStructuredContentProvider#getElements(java.lang.Object)
*/
public Object[] getElements(Object inputElement) {
if (inputElement instanceof List) {
Object[] o = ((List) inputElement).toArray();
return o;
} else {
return new Object[0];
}
}
}

View File

@@ -22,6 +22,7 @@ import org.eclipse.swt.widgets.ToolItem;
import org.talend.commons.ui.runtime.image.EImage;
import org.talend.designer.mapper.i18n.Messages;
import org.talend.designer.mapper.managers.MapperManager;
import org.talend.designer.mapper.managers.UIManager;
import org.talend.designer.mapper.ui.visualmap.table.DataMapTableView;
import org.talend.designer.mapper.ui.visualmap.zone.Zone;
@@ -109,7 +110,7 @@ public class ToolbarOutputZone extends ToolbarZone {
* DOC amaumont Comment method "addListeners".
*/
private void addListeners() {
// final UIManager uiManager = getMapperManager().getUiManager();
final UIManager uiManager = getMapperManager().getUiManager();
addOutputItem.addListener(SWT.Selection, new Listener() {
public void handleEvent(Event event) {
@@ -133,7 +134,12 @@ public class ToolbarOutputZone extends ToolbarZone {
for (DataMapTableView dataMapTableView : outputsTablesView) {
dataMapTableView.notifyFocusLost();
}
getMapperManager().mapAutomaticallly();
uiManager.openAutoMappingDialog();
// IPreferenceStore weightStore = CoreUIPlugin.getDefault().getPreferenceStore();
// getMapperManager().mapAutomaticallly(weightStore.getInt(ITalendCorePrefConstants.LEVENSHTEIN_WEIGHT),
// weightStore.getInt(ITalendCorePrefConstants.JACCARD_WEIGHT));
}
});