Transform视图界面创建及执行机制
创建Transform视图
读取*.ktr文件创建Transform视图流程
Spoon.java
private void loadLastUsedFile(
LastUsedFile lastUsedFile, String repositoryName, boolean trackIt, boolean isStartup ) throws KettleException {
if ( lastUsedFile.isTransformation() ) {
openFile( lastUsedFile.getFilename(), variables, rep != null );
}
if ( lastUsedFile.isJob() ) {
openFile( lastUsedFile.getFilename(), variables, false );
}
}
Spoon.java
public void openFile( String filename, VariableSpace variableSpace, boolean importfile ) {
// otherwise try by looking at the root node if we were able to parse file
// as XML
if ( listener == null && root != null ) {
for ( FileListener li : fileListeners ) {
if ( li.acceptsXml( root.getNodeName() ) ) {
listener = li;
break;
}
}
}
loaded = listener.open( root, filename, importfile );
}
FileListener.java
public interface FileListener {
public boolean open( Node transNode, String fname, boolean importfile ) throws KettleMissingPluginsException;
public boolean save( EngineMetaInterface meta, String fname, boolean isExport );
public void syncMetaName( EngineMetaInterface meta, String name );
public boolean accepts( String fileName );
public boolean acceptsXml( String nodeName );
public String[] getSupportedExtensions();
public String[] getFileTypeDisplayNames( Locale locale );
public String getRootNodeName();
}
TransFileListener.java
public boolean open( Node transNode, String fname, String connection, boolean importfile )
throws KettleMissingPluginsException {
final Spoon spoon = Spoon.getInstance();
TransMeta transMeta = new TransMeta();
transMeta.loadXML( transNode, fname, spoon.getMetaStore(), spoon.getRepository(), ,..)
spoon.addTransGraph( transMeta );
}
TransMeta.java loadXML 解析 *.ktr文件
public void loadXML( Node transnode, String fname, IMetaStore metaStore, Repository rep, boolean setInternalVariables,
VariableSpace parentVariableSpace, OverwritePrompter prompter )
throws KettleXMLException, KettleMissingPluginsException {
// Handle Steps
int s = XMLHandler.countNodes( transnode, StepMeta.XML_TAG );
for ( int i = 0; i < s; i++ ) {
Node stepnode = XMLHandler.getSubNodeByNr( transnode, StepMeta.XML_TAG, i );
StepMeta stepMeta = new StepMeta( stepnode, databases, metaStore );
stepMeta.setParentTransMeta( this ); // for tracing, retain hierarchy
StepMeta check = findStep( stepMeta.getName() );
if ( check != null ) {
if ( !check.isShared() ) {
addOrReplaceStep( stepMeta );
} else {
check.setDraw( stepMeta.isDrawn() ); // Just keep the drawn flag and location
check.setLocation( stepMeta.getLocation() );
}
} else {
addStep( stepMeta ); // simply add it.
}
}
// Handle Hops
//
Node ordernode = XMLHandler.getSubNode( transnode, XML_TAG_ORDER );
n = XMLHandler.countNodes( ordernode, TransHopMeta.XML_HOP_TAG );
for ( int i = 0; i < n; i++ ) {
Node hopnode = XMLHandler.getSubNodeByNr( ordernode, TransHopMeta.XML_HOP_TAG, i );
TransHopMeta hopinf = new TransHopMeta( hopnode, steps );
hopinf.setErrorHop( isErrorNode( errorHandlingNode, hopnode ) );
addTransHop( hopinf );
}
}
StepMeta.java类构造函数
public StepMeta( Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore ) throws KettleXMLException,
KettlePluginLoaderException {
this();
PluginRegistry registry = PluginRegistry.getInstance();
// Create a new StepMetaInterface object...
PluginInterface sp = registry.findPluginWithId( StepPluginType.class, stepid, true );
if ( sp == null ) {
setStepMetaInterface( new MissingTrans( name, stepid ) );
} else {
setStepMetaInterface( (StepMetaInterface) registry.loadClass( sp ) );
}
if ( this.stepMetaInterface != null ) {
if ( sp != null ) {
stepid = sp.getIds()[0]; // revert to the default in case we loaded an alternate version
}
// Load the specifics from XML...
if ( stepMetaInterface != null ) {
loadXmlCompatibleStepMeta( stepMetaInterface, stepnode, databases );
stepMetaInterface.loadXML( stepnode, databases, metaStore );
}
}
}
TextFileInputMeta类实现了*.ktr中type为TextFileInput的step的下的参数解析
public class TextFileInputMeta extends BaseFileInputMeta<BaseFileInputAdditionalField, BaseFileInputFiles, BaseFileField>
implements StepMetaInterface, ResolvableResource, CsvInputAwareMeta {
@Override
public void loadXML( Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore ) throws KettleXMLException {
try {
inputFiles.acceptingFilenames = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "accept_filenames" ) );
inputFiles.passingThruFields =
YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "passing_through_fields" ) );
inputFiles.acceptingField = XMLHandler.getTagValue( stepnode, "accept_field" );
content.separator = XMLHandler.getTagValue( stepnode, "separator" );
content.enclosure = XMLHandler.getTagValue( stepnode, "enclosure" );
String nempty = XMLHandler.getTagValue( stepnode, "noempty" );
content.noEmptyLines = YES.equalsIgnoreCase( nempty ) || nempty == null;
content.includeFilename = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "include" ) );
content.filenameField = XMLHandler.getTagValue( stepnode, "include_field" );
content.includeRowNumber = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownum" ) );
content.rowNumberByFile = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "rownumByFile" ) );
content.rowNumberField = XMLHandler.getTagValue( stepnode, "rownum_field" );
content.fileFormat = XMLHandler.getTagValue( stepnode, "format" );
content.encoding = XMLHandler.getTagValue( stepnode, "encoding" );
content.length = XMLHandler.getTagValue( stepnode, "length" );
Node filenode = XMLHandler.getSubNode( stepnode, "file" );
Node fields = XMLHandler.getSubNode( stepnode, "fields" );
Node filtersNode = XMLHandler.getSubNode( stepnode, "filters" );
int nrfiles = XMLHandler.countNodes( filenode, "name" );
int nrfields = XMLHandler.countNodes( fields, "field" );
int nrfilters = XMLHandler.countNodes( filtersNode, "filter" );
allocate( nrfiles, nrfields, nrfilters );
for ( int i = 0; i < nrfiles; i++ ) {
Node filenamenode = XMLHandler.getSubNodeByNr( filenode, "name", i );
inputFiles.fileName[i] = loadSource( filenode, filenamenode, i, metaStore );
}
content.fileType = XMLHandler.getTagValue( stepnode, "file", "type" );
content.fileCompression = XMLHandler.getTagValue( stepnode, "file", "compression" );
if ( content.fileCompression == null ) {
content.fileCompression = "None";
if ( YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "zipped" ) ) ) {
content.fileCompression = "Zip";
}
}
// Backward compatibility : just one filter
// Is there a limit on the number of rows we process?
content.rowLimit = Const.toLong( XMLHandler.getTagValue( stepnode, "limit" ), 0L );
errorHandling.errorIgnored = YES.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "error_ignored" ) );
// Backward compatible
content.dateFormatLenient = !NO.equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "date_format_lenient" ) );
additionalOutputFields.shortFilenameField = XMLHandler.getTagValue( stepnode, "shortFileFieldName" );
additionalOutputFields.pathField = XMLHandler.getTagValue( stepnode, "pathFieldName" );
} catch ( Exception e ) {
throw new KettleXMLException( "Unable to load step info from XML", e );
}
}
}
@InjectionSupported( localizationPrefix = "TextFileOutput.Injection.", groups = { "OUTPUT_FIELDS" } )
public class TextFileOutputMeta extends BaseFileOutputMeta implements StepMetaInterface, ResolvableResource {
@Override
public void loadXML( Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore ) throws KettleXMLException {
readData( stepnode, metaStore );
}
protected void readData( Node stepnode, IMetaStore metastore ) throws KettleXMLException {
try {
separator = XMLHandler.getTagValue( stepnode, "separator" );
if ( separator == null ) {
separator = "";
}
enclosure = XMLHandler.getTagValue( stepnode, "enclosure" );
if ( enclosure == null ) {
enclosure = "";
}
enclosureForced = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "enclosure_forced" ) );
headerEnabled = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "header" ) );
footerEnabled = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "footer" ) );
fileFormat = XMLHandler.getTagValue( stepnode, "format" );
setFileCompression( XMLHandler.getTagValue( stepnode, "compression" ) );
fileName = loadSource( stepnode, metastore );
servletOutput = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "servlet_output" ) );
doNotOpenNewFileInit =
"Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "do_not_open_new_file_init" ) );
extension = XMLHandler.getTagValue( stepnode, "file", "extention" );
fileAppended = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "append" ) );
stepNrInFilename = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "split" ) );
partNrInFilename = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "file", "haspartno" ) );
fileNameInField = "Y".equalsIgnoreCase( XMLHandler.getTagValue( stepnode, "fileNameInField" ) );
fileNameField = XMLHandler.getTagValue( stepnode, "fileNameField" );
Node fields = XMLHandler.getSubNode( stepnode, "fields" );
int nrfields = XMLHandler.countNodes( fields, "field" );
allocate( nrfields );
for ( int i = 0; i < nrfields; i++ ) {
Node fnode = XMLHandler.getSubNodeByNr( fields, "field", i );
outputFields[i] = new TextFileField();
outputFields[i].setName( XMLHandler.getTagValue( fnode, "name" ) );
}
} catch ( Exception e ) {
throw new KettleXMLException( "Unable to load step info from XML", e );
}
}
}
拖动插件到Transform视图区流程
类型为DragAndDropContainer.TYPE_BASE_STEP_TYPE
TransGraph.java
@Override
public void drop( DropTargetEvent event ) {
switch ( container.getType() ) {
// Put an existing one on the canvas.
case DragAndDropContainer.TYPE_STEP:
// Drop hidden step onto canvas....
stepMeta = transMeta.findStep( container.getData() );
break;
// Create a new step
case DragAndDropContainer.TYPE_BASE_STEP_TYPE:
stepMeta = spoon.newStep( transMeta, id, name, name, false, true );
break;
// Create a new TableInput step using the selected connection...
case DragAndDropContainer.TYPE_DATABASE_CONNECTION:
newstep = true;
String connectionName = container.getData();
TableInputMeta tii = new TableInputMeta();
tii.setDatabaseMeta( transMeta.findDatabase( connectionName ) );
PluginRegistry registry = PluginRegistry.getInstance();
String stepID = registry.getPluginId( StepPluginType.class, tii );
PluginInterface stepPlugin = registry.findPluginWithId( StepPluginType.class, stepID );
String stepName = transMeta.getAlternativeStepname( stepPlugin.getName() );
stepMeta = new StepMeta( stepID, stepName, tii );
if ( spoon.editStep( transMeta, stepMeta ) != null ) {
transMeta.addStep( stepMeta );
spoon.refreshTree();
spoon.refreshGraph();
} else {
return;
}
break;
// Drag hop on the canvas: create a new Hop...
case DragAndDropContainer.TYPE_TRANS_HOP:
newHop();
return;
default:
// Nothing we can use: give an error!
modalMessageDialog( getString( "TransGraph.Dialog.ItemCanNotBePlacedOnCanvas.Title" ),
getString( "TransGraph.Dialog.ItemCanNotBePlacedOnCanvas.Message" ), SWT.OK );
return;
}
stepMeta.drawStep();
stepMeta.setSelected( true );
}
双击插件到Transform视图区流程
Spoon.java
private void doubleClickedInTree( Tree tree, boolean shift ) {
if ( selection instanceof PluginInterface ) {
PluginInterface plugin = (PluginInterface) selection;
if ( plugin.getPluginType().equals( StepPluginType.class ) ) {
TransGraph transGraph = getActiveTransGraph();
if ( transGraph != null ) {
transGraph.addStepToChain( plugin, shift );
}
}
if ( plugin.getPluginType().equals( JobEntryPluginType.class ) ) {
JobGraph jobGraph = getActiveJobGraph();
if ( jobGraph != null ) {
jobGraph.addJobEntryToChain( object.getItemText(), shift );
}
}
}
}
}
TransGraph.java
TransGraph.java
public void addStepToChain( PluginInterface stepPlugin, boolean shift ) {
TransMeta transMeta = spoon.getActiveTransformation();
StepMeta newStep =
spoon.newStep( transMeta, stepPlugin.getIds()[ 0 ], stepPlugin.getName(), stepPlugin.getName(), false, true );
TransHopMeta hop = new TransHopMeta( lastChained, newStep );
spoon.newHop( transMeta, hop );
}
Spoon.java
public StepMeta newStep( TransMeta transMeta, String id, String name, String description, boolean openit, boolean rename ) {
PluginRegistry registry = PluginRegistry.getInstance();
PluginInterface stepPlugin = id != null ? registry.findPluginWithId( StepPluginType.class, id )
: registry.findPluginWithName( StepPluginType.class, description );
if ( stepPlugin != null ) {
StepMetaInterface info = (StepMetaInterface) registry.loadClass( stepPlugin );
info.setDefault();
if ( openit ) {
StepDialogInterface dialog = this.getStepEntryDialog( info, transMeta, name );
if ( dialog != null ) {
name = dialog.open();
}
}
inf = new StepMeta( stepPlugin.getIds()[0], name, info );
if ( name != null ) {
// OK pressed in the dialog: we have a step-name
String newName = name;
StepMeta stepMeta = transMeta.findStep( newName );
int nr = 2;
while ( stepMeta != null ) {
newName = name + " " + nr;
stepMeta = transMeta.findStep( newName );
nr++;
}
transMeta.addStep( inf );
refreshTree();
}
return inf;
}
Transform的Run机制
Transform的Run界面触发
TransGraph.java
private void addToolBar() {
// Added 1/11/2016 to implement dropdown option for "Run"
ToolItem runItem = new ToolItem( swtToolbar, SWT.DROP_DOWN, 0 );
runItem.addSelectionListener( new SelectionAdapter() {
@Override
public void widgetSelected( SelectionEvent e ) {
if ( e.detail == SWT.DROP_DOWN ) {
Menu menu = new Menu( shell, SWT.POP_UP );
MenuItem item1 = new MenuItem( menu, SWT.PUSH );
item1.addSelectionListener( new SelectionAdapter() {
@Override
public void widgetSelected( SelectionEvent e1 ) {
runTransformation();
}
} );
MenuItem item2 = new MenuItem( menu, SWT.PUSH );
item2.addSelectionListener( new SelectionAdapter() {
@Override
public void widgetSelected( SelectionEvent e2 ) {
runOptionsTransformation();
}
} );
} else {
runTransformation();
}
}
//类似有
stopTransformation();
｝
Transform的Run执行机制
TransGraph.java
public void runTransformation() {
spoon.runFile();
}
Spoon.java
public void runFile() {
executeFile( true, false, false, false, false, null, false, false );
}
Spoon.java
jpublic void executeFile( boolean local, boolean remote, boolean cluster, boolean preview, boolean debug,
Date replayDate, boolean safe, boolean show ) {
TransMeta transMeta = getActiveTransformation();
if ( transMeta != null ) {
executeTransformation( transMeta, local, remote, cluster, preview, debug, replayDate,
safe,transExecutionConfiguration.getLogLevel() );
}
JobMeta jobMeta = getActiveJob();
if ( jobMeta != null ) {
executeJob( jobMeta, local, remote, replayDate, safe, null, 0 );
}
}
Spoon.java
public void executeTransformation( final TransMeta transMeta, final boolean local, final boolean remote,
final boolean cluster, final boolean preview, final boolean debug, final Date replayDate,
final boolean safe, final LogLevel logLevel ) {
if ( RepositorySecurityUI.verifyOperations( shell, rep, RepositoryOperation.EXECUTE_TRANSFORMATION ) ) {
return;
}
Thread thread = new Thread() {
@Override
public void run() {
getDisplay().asyncExec( new Runnable() {
@Override
public void run() {
try {
delegates.trans.executeTransformation(
transMeta, local, remote, cluster, preview, debug, replayDate, safe, logLevel );
} catch ( Exception e ) {
new ErrorDialog(
shell, "Execute transformation", "There was an error during transformation execution", e );
}
}
} );
}
};
thread.start();
}
SpoonTransformationDelegate.java
public class SpoonTransformationDelegate extends SpoonDelegate {
public void executeTransformation( final TransMeta transMeta, final boolean local, final boolean remote,
final boolean cluster, final boolean preview, final boolean debug, final Date replayDate, final boolean safe,
LogLevel logLevel ) throws KettleException {
// Is this a local execution?
//
if ( executionConfiguration.isExecutingLocally() ) {
if ( debug || preview ) {
activeTransGraph.debug( executionConfiguration, transDebugMeta );
} else {
activeTransGraph.start( executionConfiguration );
}
// Are we executing remotely?
//
} else if ( executionConfiguration.isExecutingRemotely() ) {}
}
}
TransGraph.java
public synchronized void start( TransffExecutionConfiguration executionConfiguration ) throws KettleException
{
final Thread parentThread = Thread.currentThread();
shell.getDisplay().asyncExec( new Runnable() {
@Override
public void run() {
addAllTabs();
prepareTrans( parentThread, args );
}
} );
}
}
TransGraph.java
public void addAllTabs() {
transHistoryDelegate.addTransHistory();
transLogDelegate.addTransLog();
transGridDelegate.addTransGrid();
transPerfDelegate.addTransPerf();
transMetricsDelegate.addTransMetrics();
transPreviewDelegate.addTransPreview();
}
private synchronized void prepareTrans( final Thread parentThread, final String[] args ) {
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
trans.prepareExecution( args );
TransGraph.java
private synchronized void prepareTrans( final Thread parentThread, final String[] args ) {
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
trans.prepareExecution( args );
// Do we capture data?
//
if ( transPreviewDelegate.isActive() ) {
transPreviewDelegate.capturePreviewData( trans, transMeta.getSteps() );
}
initialized = true;
} catch ( KettleException e ) {
log.logError( trans.getName() + ": preparing transformation execution failed", e );
checkErrorVisuals();
}
halted = trans.hasHaltedSteps();
if ( trans.isReadyToStart() ) {
checkStartThreads(); // After init, launch the threads.
} else {
initialized = false;
running = false;
checkErrorVisuals();
}
}
};
Thread thread = new Thread( runnable );
thread.start();
}
private void checkStartThreads() {
if ( initialized && !running && trans != null ) {
startThreads();
}
}
private synchronized void startThreads() {
trans.startThreads();
}
Trans.java
public void startThreads() throws KettleException {
switch ( transMeta.getTransformationType() ) {
case Normal:
// Now start all the threads...
for ( int i = 0; i < steps.size(); i++ ) {
final StepMetaDataCombi combi = steps.get( i );
RunThread runThread = new RunThread( combi );
Thread thread = new Thread( runThread );
thread.setName( getName() + " - " + combi.stepname );
} );
thread.start();
}
break;
case SerialSingleThreaded:
break;
case SingleThreaded:
break;
default:
break;
}
}
RunThread.java中的run方法
public class RunThread implements Runnable {
public RunThread( StepMetaDataCombi combi ) {
this.step = combi.step;
this.meta = combi.meta;
this.data = combi.data;
this.log = step.getLogChannel();
}
public void run() {
step.setRunning( true );
// Wait
while ( step.processRow( meta, data ) ) {
if ( step.isStopped() ) {
break;
}
}
}
}
public void capturePreviewData( final Trans trans, List<StepMeta> stepMetas ) {
final TransMeta transMeta = trans.getTransMeta();
for ( final StepMeta stepMeta : stepMetas ) {
try {
StepInterface step = trans.findRunThread( stepMeta.getName() );
if ( step != null ) {
switch ( previewMode ) {
case LAST:
step.addRowListener( new RowAdapter() {
@Override
public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
try {
rowsData.add( new RowMetaAndData( rowMeta, rowMeta.cloneRow( row ) ) );
if ( rowsData.size() > PropsUI.getInstance().getDefaultPreviewSize() ) {
rowsData.remove( 0 );
}
} catch ( Exception e ) {
throw new KettleStepException( "Unable to clone row for metadata : " + rowMeta, e );
}
}
} );
break;
default:
step.addRowListener( new RowAdapter() {
@Override
public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
if ( rowsData.size() < PropsUI.getInstance().getDefaultPreviewSize() ) {
try {
rowsData.add( new RowMetaAndData( rowMeta, rowMeta.cloneRow( row ) ) );
} catch ( Exception e ) {
throw new KettleStepException( "Unable to clone row for metadata : " + rowMeta, e );
}
}
}
} );
break;
}
}
} catch ( Exception e ) {
loggingText.append( Const.getStackTracker( e ) );
}
}
} );
}
Transform的Debug机制
Transform的Previewer机制
浙公网安备 33010602011771号