kettle 同步组件 synchronizeAfterMerge JAVA API

package com.example.fg.kettle;


import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.ObjectLocationSpecificationMethod;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.KettleLogStore;
import org.pentaho.di.job.Job;
import org.pentaho.di.job.JobHopMeta;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.job.entries.special.JobEntrySpecial;
import org.pentaho.di.job.entries.success.JobEntrySuccess;
import org.pentaho.di.job.entries.trans.JobEntryTrans;
import org.pentaho.di.job.entry.JobEntryCopy;
import org.pentaho.di.repository.ObjectId;
import org.pentaho.di.repository.RepositoryDirectoryInterface;
import org.pentaho.di.repository.RepositoryElementInterface;
import org.pentaho.di.repository.kdr.KettleDatabaseRepository;
import org.pentaho.di.repository.kdr.KettleDatabaseRepositoryMeta;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransHopMeta;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepIOMetaInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.errorhandling.StreamInterface;
import org.pentaho.di.trans.steps.insertupdate.InsertUpdateMeta;
import org.pentaho.di.trans.steps.mergerows.MergeRows;
import org.pentaho.di.trans.steps.mergerows.MergeRowsMeta;
import org.pentaho.di.trans.steps.synchronizeaftermerge.SynchronizeAfterMerge;
import org.pentaho.di.trans.steps.synchronizeaftermerge.SynchronizeAfterMergeMeta;
import org.pentaho.di.trans.steps.tableinput.TableInputMeta;

import java.util.List;

public class KettleJobSynchronizeAfterMerge {
    public static void main(String[] args) throws Exception {
        runSyncJob();
//        runSyncLoadDBJob();
//        runSyncLoadDBTrans();

    }

    public static void runSyncJob() throws KettleException {
        String transName = "testTransName1";
        String repositoryName = "cgmRepository";
        String jobName = "testJobName1";
        String sourceTableName = "xy_social_security";
        String targetTableName = "xy_social_security_qfzx_test";
        String targetDatabase = "fromDbName";
        String sourceDatabase = "toDbName";
        String[] columns = new String[]{"id", "eid", "name", "report_year", "report_name", "report_date", "bq_shengyubx_je", "dw_je_display", "dw_yanglaobx_je", "bq_shiyebx_je", "dw_yiliaobx_je", "shiyebx_num", "dw_yanglaobx_js", "dw_gongshangbx_js", "yiliaobx_num", "dw_shengyubx_je", "dw_js_display", "dw_shengyubx_js", "bq_yiliaobx_je", "bq_gongshangbx_je", "dw_gongshangbx_je", "shengyubx_num", "dw_shiyebx_js", "dw_shiyebx_je", "bq_je_display", "gongshangbx_num", "yanglaobx_num", "bq_yanglaobx_je", "dw_yiliaobx_js", "currency", "created_time", "row_update_time", "local_row_update_time"};
        KettleDatabaseRepository kettleDatabaseRepository = RepositoryCon(repositoryName);
        RepositoryDirectoryInterface dir = kettleDatabaseRepository.loadRepositoryDirectoryTree().findDirectory("/");
        ObjectId objectId = kettleDatabaseRepository.getTransformationID(transName, dir);
        TransMeta transMeta;
        if (objectId == null) {
            transMeta = new TransMeta();
            transMeta.setName(transName);
            DatabaseMeta sourceDatabaseMeta = new DatabaseMeta(targetDatabase, "mysql", "Native(JDBC)", "127.0.0.1", "test?useSSL=false&serverTimezone=Asia/Shanghai", "3306", "root", "toor@1234");
            transMeta.addDatabase(sourceDatabaseMeta);
            DatabaseMeta targetDatabaseMeta = new DatabaseMeta(sourceDatabase, "mysql", "Native(JDBC)", "127.0.0.1", "test?useSSL=false&serverTimezone=Asia/Shanghai", "3306", "root", "toor@1234");
            transMeta.addDatabase(targetDatabaseMeta);

            TableInputMeta oldTableInput = new TableInputMeta();
            DatabaseMeta database_bjdt = transMeta.findDatabase(sourceDatabase);
            oldTableInput.setDatabaseMeta(database_bjdt);
            String old_select_sql = "SELECT * FROM " + sourceTableName + " order by id";
            oldTableInput.setSQL(old_select_sql);

            StepMeta oldTableInputMetaStep = new StepMeta("源数据", oldTableInput);
            oldTableInputMetaStep.setDraw(true);
            oldTableInputMetaStep.setLocation(100, 300);
            transMeta.addStep(oldTableInputMetaStep);

            TableInputMeta newTableInput = new TableInputMeta();
            DatabaseMeta database_kettle = transMeta.findDatabase(targetDatabase);
            newTableInput.setDatabaseMeta(database_kettle);
            String new_select_sql = "SELECT * FROM " + targetTableName + " order by id";
            newTableInput.setSQL(new_select_sql);

            StepMeta newTableInputMetaStep = new StepMeta("目标数据", newTableInput);
            newTableInputMetaStep.setDraw(true);
            newTableInputMetaStep.setLocation(100, 200);
            transMeta.addStep(newTableInputMetaStep);


            MergeRowsMeta mergeRowsMeta = new MergeRowsMeta();
            StepIOMetaInterface stepIOMeta = mergeRowsMeta.getStepIOMeta();
            stepIOMeta.getInfoStreams().get(0).setStepMeta(newTableInputMetaStep);
            stepIOMeta.getInfoStreams().get(1).setStepMeta(oldTableInputMetaStep);
            mergeRowsMeta.setFlagField("bz");
            mergeRowsMeta.setKeyFields(new String[]{"ID"});
            mergeRowsMeta.setValueFields(columns);
            StepMeta mergeStepMeta = new StepMeta("合并记录", mergeRowsMeta);
            mergeStepMeta.setDraw(true);
            mergeStepMeta.setLocation(300, 400);
            transMeta.addStep(mergeStepMeta);

            transMeta.addTransHop(new TransHopMeta(oldTableInputMetaStep, mergeStepMeta));
            transMeta.addTransHop(new TransHopMeta(newTableInputMetaStep, mergeStepMeta));

            SynchronizeAfterMergeMeta synchronizeAfterMergeMeta = new SynchronizeAfterMergeMeta();
            synchronizeAfterMergeMeta.setCommitSize(10000);
            synchronizeAfterMergeMeta.setDatabaseMeta(targetDatabaseMeta);
            synchronizeAfterMergeMeta.setSchemaName("");
            synchronizeAfterMergeMeta.setTableName(targetTableName);
            synchronizeAfterMergeMeta.setUseBatchUpdate(true);
            synchronizeAfterMergeMeta.setKeyLookup(new String[]{"id"});
            synchronizeAfterMergeMeta.setKeyStream(new String[]{"id"});
            synchronizeAfterMergeMeta.setKeyStream2(new String[]{""});
            synchronizeAfterMergeMeta.setKeyCondition(new String[]{"="});

            Boolean[] updateOrNot = new Boolean[columns.length];
            for (int i = 0; i < columns.length; i++) {
                updateOrNot[i] = true;
            }
            updateOrNot[0] = false;
            synchronizeAfterMergeMeta.setUpdateLookup(columns);
            synchronizeAfterMergeMeta.setUpdateStream(columns);
            synchronizeAfterMergeMeta.setUpdate(updateOrNot);

            synchronizeAfterMergeMeta.setOperationOrderField("bz");
            synchronizeAfterMergeMeta.setOrderInsert("new");
            synchronizeAfterMergeMeta.setOrderUpdate("changed");
            synchronizeAfterMergeMeta.setOrderDelete("deleted");
            StepMeta synStepMeta = new StepMeta("数据同步", synchronizeAfterMergeMeta);
            synStepMeta.setDraw(true);
            synStepMeta.setLocation(500, 600);
            transMeta.addStep(synStepMeta);

            transMeta.addTransHop(new TransHopMeta(mergeStepMeta, synStepMeta));


            transMeta.setRepositoryDirectory(dir);
            kettleDatabaseRepository.save(transMeta, null);
        }
        ObjectId id = kettleDatabaseRepository.getJobId(jobName, dir);
        if (id == null) {
            JobMeta jobMeta = new JobMeta();
            jobMeta.setName(jobName);
            jobMeta.setJobstatus(0);

            JobEntrySpecial jobEntrySpecial = new JobEntrySpecial();
            jobEntrySpecial.setName("START");
            jobEntrySpecial.setStart(true);
//            jobEntrySpecial.setRepeat(true);
//            jobEntrySpecial.setIntervalSeconds(3);
            JobEntryCopy start = new JobEntryCopy(jobEntrySpecial);
            start.setDrawn();
            start.setLocation(100, 200);

            JobEntryTrans jobEntryTrans = new JobEntryTrans();
            ObjectLocationSpecificationMethod SpecMethod = ObjectLocationSpecificationMethod.getSpecificationMethodByCode(repositoryName);
            jobEntryTrans.setSpecificationMethod(SpecMethod);
            jobEntryTrans.setRepository(kettleDatabaseRepository);
            jobEntryTrans.setDirectory("/");
            jobEntryTrans.setTransname(transName);
            JobEntryCopy trans1 = new JobEntryCopy(jobEntryTrans);
            trans1.setName("JTrans");
            trans1.setDrawn(true);
            trans1.setLocation(200, 300);

            JobEntrySuccess jobEntrySuccess = new JobEntrySuccess();
            jobEntrySuccess.setName("Success");
            JobEntryCopy success = new JobEntryCopy(jobEntrySuccess);
            success.setDrawn();
            success.setLocation(300, 400);

            jobMeta.addJobEntry(start);
            jobMeta.addJobEntry(trans1);
            jobMeta.addJobEntry(success);

            jobMeta.addJobHop(new JobHopMeta(start, trans1));
            jobMeta.addJobHop(new JobHopMeta(trans1, success));

            jobMeta.setRepositoryDirectory(dir);
            kettleDatabaseRepository.save(jobMeta, null);
            id = kettleDatabaseRepository.getJobId(jobName, dir);
        }

        JobMeta jobMetaObj = kettleDatabaseRepository.loadJob(id, null);
        Job job = new Job(kettleDatabaseRepository, jobMetaObj);
        job.run();
        job.waitUntilFinished();
        String strLog = KettleLogStore.getAppender().getBuffer().toString();
        System.out.println("==========开始打印日志==========");

        System.out.println(KettleLogStore.getAppender().getBuffer().toString());
        System.out.println("==========日志打印结束==========");
        System.out.println("getLastProcessed:" + job.getStatus());

        String substring = strLog.substring(strLog.lastIndexOf("I=") + 1);
        String successCount = substring.substring(substring.lastIndexOf("W=") + 2, substring.lastIndexOf("W=") + 3);
        System.out.println("成功数：" + successCount);

        System.out.println("errors:" + job.getErrors());
        if (job.getErrors() != 0) {
            System.out.println("执行失败！");
        }


    }

    public static KettleDatabaseRepository RepositoryCon(String repositoryName) throws KettleException {
        // 初始化环境
        if (!KettleEnvironment.isInitialized()) {
            try {
                KettleEnvironment.init();
            } catch (KettleException e) {
                e.printStackTrace();
            }
        }
        DatabaseMeta databaseMeta = new DatabaseMeta(repositoryName, "mysql", "Native(JDBC)", "127.0.0.1", "test?useSSL=false&serverTimezone=Asia/Shanghai", "3306", "root", "toor@1234");
        KettleDatabaseRepositoryMeta kettleDatabaseRepositoryMeta = new KettleDatabaseRepositoryMeta();
        kettleDatabaseRepositoryMeta.setConnection(databaseMeta);
        KettleDatabaseRepository kettleDatabaseRepository = new KettleDatabaseRepository();
        kettleDatabaseRepository.init(kettleDatabaseRepositoryMeta);
        kettleDatabaseRepository.connect("admin", "admin");
        if (kettleDatabaseRepository.isConnected()) {
            System.out.println("连接成功");
            return kettleDatabaseRepository;
        } else {
            System.out.println("连接失败");
            return null;
        }
    }

    public static void runSyncLoadDBJob() throws KettleException {
        String repositoryName = "test-db";
        String jobName = "cgmJobName";
        KettleDatabaseRepository kettleDatabaseRepository = RepositoryCon(repositoryName);
        RepositoryDirectoryInterface dir = kettleDatabaseRepository.loadRepositoryDirectoryTree().findDirectory("/");
        ObjectId id = kettleDatabaseRepository.getJobId(jobName, dir);
        JobMeta jobMetaObj = kettleDatabaseRepository.loadJob(id, null);
        Job job = new Job(kettleDatabaseRepository, jobMetaObj);
        job.run();
        job.waitUntilFinished();
        String strLog = KettleLogStore.getAppender().getBuffer().toString();
        System.out.println("==========开始打印日志==========");

        System.out.println(KettleLogStore.getAppender().getBuffer().toString());
        System.out.println("==========日志打印结束==========");
        System.out.println("getLastProcessed:" + job.getStatus());

        String substring = strLog.substring(strLog.lastIndexOf("I=") + 1);
        String successCount = substring.substring(substring.lastIndexOf("W=") + 2, substring.lastIndexOf("W=") + 3);
        System.out.println("成功数：" + successCount);

        System.out.println("errors:" + job.getErrors());
        if (job.getErrors() != 0) {
            System.out.println("执行失败！");
        }
    }

    public static void runSyncLoadDBTrans() throws KettleException {
        KettleDatabaseRepository kettleDatabaseRepository = RepositoryCon("cgmRepository");
        RepositoryDirectoryInterface dir = kettleDatabaseRepository.loadRepositoryDirectoryTree().findDirectory("/");
        ObjectId objectId = kettleDatabaseRepository.getTransformationID("Test_tran_sync", dir);
        TransMeta transMeta = kettleDatabaseRepository.loadTransformation(objectId, null);
        Trans trans = new Trans(transMeta);
        trans.execute(null);
        trans.waitUntilFinished();
    }


}