随笔- 70 文章- 0 评论- 1 阅读- 13万

分布式定时任务-利用分布式定时任务框架xxl-job实现任务动态发布

1.场景：项目前期使用k8s部署的单节点，后期生产需要将单节点的服务扩展多个节点，每个节点的定时任务使用的quartz实现，如果不加限制且定时任务有对数据库的写操作，在不同节点上执行的定时任务容易造成数据库产生脏数据，所以需要分布式任务框架对任务进行控制，这里我们使用xxl-job实现。

2.需要下载并部署xxl-job-admin，并配置相关的数据库，启动xxl-job-admin ，在项目定时任务所在的服务引入xxl-job-core,配置并改造需要的定时任务

3.application.yml配置

xxl:
  job:
    admin:
      addresses: http://127.0.0.1:8080/xxl-job-admin
    accessToken:
    executor:
      appname: customer
      address:
      ip: 127.0.0.1
      port: 9999
      logpath: /data/applogs/xxl-job/jobhandler
      logretentiondays: 30

4.在调度中心创建执行器与任务

在执行器管理中创建执行器

执行器的名称要和配置文件中的appname一致

切换到任务管理，创建人物

创建任务映射

5.build.gradle引入xxl-job-core

dependencies {
    // https://mvnrepository.com/artifact/com.xuxueli/xxl-job-core
    implementation group: 'com.xuxueli', name: 'xxl-job-core', version: '2.3.0'
}

6.添加xxl-job-core的配置类

import com.xxl.job.core.executor.impl.XxlJobSpringExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
 
/**
 * xxl-job config
 *
 * @author xuxueli 2017-04-28
 */
//@Configuration
public class XxlJobConfig {
    private Logger logger = LoggerFactory.getLogger(XxlJobConfig.class);
 
    @Value("${xxl.job.admin.addresses}")
    private String adminAddresses;
 
    @Value("${xxl.job.accessToken}")
    private String accessToken;
 
    @Value("${xxl.job.executor.appname}")
    private String appname;
 
    @Value("${xxl.job.executor.address}")
    private String address;
 
    @Value("${xxl.job.executor.ip}")
    private String ip;
 
    @Value("${xxl.job.executor.port}")
    private int port;
 
    @Value("${xxl.job.executor.logpath}")
    private String logPath;
 
    @Value("${xxl.job.executor.logretentiondays}")
    private int logRetentionDays;
 
 
    @Bean
    public XxlJobSpringExecutor xxlJobExecutor() {
        logger.info(">>>>>>>>>>> xxl-job config init.");
        XxlJobSpringExecutor xxlJobSpringExecutor = new XxlJobSpringExecutor();
        xxlJobSpringExecutor.setAdminAddresses(adminAddresses);
        xxlJobSpringExecutor.setAppname(appname);
        xxlJobSpringExecutor.setAddress(address);
        xxlJobSpringExecutor.setIp(ip);
        xxlJobSpringExecutor.setPort(port);
        xxlJobSpringExecutor.setAccessToken(accessToken);
        xxlJobSpringExecutor.setLogPath(logPath);
        xxlJobSpringExecutor.setLogRetentionDays(logRetentionDays);
 
        return xxlJobSpringExecutor;
    }
 
    /**
     * 针对多网卡、容器内部署等情况，可借助 "spring-cloud-commons" 提供的 "InetUtils" 组件灵活定制注册IP；
     *
     *      1、引入依赖：
     *          <dependency>
     *             <groupId>org.springframework.cloud</groupId>
     *             <artifactId>spring-cloud-commons</artifactId>
     *             <version>${version}</version>
     *         </dependency>
     *
     *      2、配置文件，或者容器启动变量
     *          spring.cloud.inetutils.preferred-networks: 'xxx.xxx.xxx.'
     *
     *      3、获取IP
     *          String ip_ = inetUtils.findFirstNonLoopbackHostInfo().getIpAddress();
     */
 
 
}

7.根据需求添加任务

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

import cn.togeek.jooq.tables.records.TROrganizationRecord;
import cn.togeek.jooq.tables.records.TSOrganizationRecord;
import cn.togeek.service.util.Rest;
import cn.togeek.tools.UUIDUtil;
import cn.togeek.util.DBvisitor;
import cn.togeek.util.OrgType;
import cn.togeek.util.ServiceUrl;
import com.xxl.job.core.context.XxlJobHelper;
import com.xxl.job.core.handler.annotation.XxlJob;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
 
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Timestamp;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
 
import static cn.togeek.jooq.Tables.T_R_ORGANIZATION;
import static cn.togeek.jooq.Tables.T_S_ORGANIZATION;
 
/**
 * XxlJob开发示例（Bean模式）
 *
 * 开发步骤：
 *      1、任务开发：在Spring Bean实例中，开发Job方法；
 *      2、注解配置：为Job方法添加注解 "@XxlJob(value="自定义jobhandler名称", init = "JobHandler初始化方法", destroy = "JobHandler销毁方法")"，注解value值对应的是调度中心新建任务的JobHandler属性的值。
 *      3、执行日志：需要通过 "XxlJobHelper.log" 打印执行日志；
 *      4、任务结果：默认任务结果为 "成功" 状态，不需要主动设置；如有诉求，比如设置任务结果为失败，可以通过 "XxlJobHelper.handleFail/handleSuccess" 自主设置任务结果；
 *
 * @author xuxueli 2019-12-11 21:52:51
 */
//@Component
public class SampleXxlJob {
    private static Logger logger = LoggerFactory.getLogger(SampleXxlJob.class);
    @Autowired
    private cn.togeek.util.DBvisitor DBvisitor;
 
    /**
     * 1、简单任务示例（Bean模式）
     */
    @XxlJob("demoJobHandler")
    public void demoJobHandler() throws Exception {
        XxlJobHelper.log("XXL-JOB, Hello World.");
        System.err.println("customer job");
        for (int i = 0; i < 5; i++) {
            XxlJobHelper.log("beat at:" + i);
            TimeUnit.SECONDS.sleep(2);
        }
        // default success
    }
 
    @XxlJob("customerSycJob")
    public void sycdata() throws Exception {
        System.out.println(111);
        syncTsorganizationTrorganizationTorganizationIndb();
    }
 
    /**
     * 2、分片广播任务
     */
    @XxlJob("shardingJobHandler")
    public void shardingJobHandler() throws Exception {
 
        // 分片参数
        int shardIndex = XxlJobHelper.getShardIndex();
        int shardTotal = XxlJobHelper.getShardTotal();
 
        XxlJobHelper.log("分片参数：当前分片序号 = {}, 总分片数 = {}", shardIndex, shardTotal);
 
        // 业务逻辑
        for (int i = 0; i < shardTotal; i++) {
            if (i == shardIndex) {
                XxlJobHelper.log("第 {} 片, 命中分片开始处理", i);
            } else {
                XxlJobHelper.log("第 {} 片, 忽略", i);
            }
        }
 
    }
 
 
    /**
     * 3、命令行任务
     */
    @XxlJob("commandJobHandler")
    public void commandJobHandler() throws Exception {
        String command = XxlJobHelper.getJobParam();
        int exitValue = -1;
 
        BufferedReader bufferedReader = null;
        try {
            // command process
            ProcessBuilder processBuilder = new ProcessBuilder();
            processBuilder.command(command);
            processBuilder.redirectErrorStream(true);
 
            Process process = processBuilder.start();
            //Process process = Runtime.getRuntime().exec(command);
 
            BufferedInputStream bufferedInputStream = new BufferedInputStream(process.getInputStream());
            bufferedReader = new BufferedReader(new InputStreamReader(bufferedInputStream));
 
            // command log
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                XxlJobHelper.log(line);
            }
 
            // command exit
            process.waitFor();
            exitValue = process.exitValue();
        } catch (Exception e) {
            XxlJobHelper.log(e);
        } finally {
            if (bufferedReader != null) {
                bufferedReader.close();
            }
        }
 
        if (exitValue == 0) {
            // default success
        } else {
            XxlJobHelper.handleFail("command exit value("+exitValue+") is failed");
        }
 
    }
 
 
    /**
     * 4、跨平台Http任务
     *  参数示例：
     *      "url: http://www.baidu.com\n" +
     *      "method: get\n" +
     *      "data: content\n";
     */
    @XxlJob("httpJobHandler")
    public void httpJobHandler() throws Exception {
 
        // param parse
        String param = XxlJobHelper.getJobParam();
        if (param==null || param.trim().length()==0) {
            XxlJobHelper.log("param["+ param +"] invalid.");
 
            XxlJobHelper.handleFail();
            return;
        }
 
        String[] httpParams = param.split("\n");
        String url = null;
        String method = null;
        String data = null;
        for (String httpParam: httpParams) {
            if (httpParam.startsWith("url:")) {
                url = httpParam.substring(httpParam.indexOf("url:") + 4).trim();
            }
            if (httpParam.startsWith("method:")) {
                method = httpParam.substring(httpParam.indexOf("method:") + 7).trim().toUpperCase();
            }
            if (httpParam.startsWith("data:")) {
                data = httpParam.substring(httpParam.indexOf("data:") + 5).trim();
            }
        }
 
        // param valid
        if (url==null || url.trim().length()==0) {
            XxlJobHelper.log("url["+ url +"] invalid.");
 
            XxlJobHelper.handleFail();
            return;
        }
        if (method==null || !Arrays.asList("GET", "POST").contains(method)) {
            XxlJobHelper.log("method["+ method +"] invalid.");
 
            XxlJobHelper.handleFail();
            return;
        }
        boolean isPostMethod = method.equals("POST");
 
        // request
        HttpURLConnection connection = null;
        BufferedReader bufferedReader = null;
        try {
            // connection
            URL realUrl = new URL(url);
            connection = (HttpURLConnection) realUrl.openConnection();
 
            // connection setting
            connection.setRequestMethod(method);
            connection.setDoOutput(isPostMethod);
            connection.setDoInput(true);
            connection.setUseCaches(false);
            connection.setReadTimeout(5 * 1000);
            connection.setConnectTimeout(3 * 1000);
            connection.setRequestProperty("connection", "Keep-Alive");
            connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
            connection.setRequestProperty("Accept-Charset", "application/json;charset=UTF-8");
 
            // do connection
            connection.connect();
 
            // data
            if (isPostMethod && data!=null && data.trim().length()>0) {
                DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream());
                dataOutputStream.write(data.getBytes("UTF-8"));
                dataOutputStream.flush();
                dataOutputStream.close();
            }
 
            // valid StatusCode
            int statusCode = connection.getResponseCode();
            if (statusCode != 200) {
                throw new RuntimeException("Http Request StatusCode(" + statusCode + ") Invalid.");
            }
 
            // result
            bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8"));
            StringBuilder result = new StringBuilder();
            String line;
            while ((line = bufferedReader.readLine()) != null) {
                result.append(line);
            }
            String responseMsg = result.toString();
 
            XxlJobHelper.log(responseMsg);
 
            return;
        } catch (Exception e) {
            XxlJobHelper.log(e);
 
            XxlJobHelper.handleFail();
            return;
        } finally {
            try {
                if (bufferedReader != null) {
                    bufferedReader.close();
                }
                if (connection != null) {
                    connection.disconnect();
                }
            } catch (Exception e2) {
                XxlJobHelper.log(e2);
            }
        }
 
    }
 
    /**
     * 5、生命周期任务示例：任务初始化与销毁时，支持自定义相关逻辑；
     */
    @XxlJob(value = "demoJobHandler2", init = "init", destroy = "destroy")
    public void demoJobHandler2() throws Exception {
        XxlJobHelper.log("XXL-JOB, Hello World.");
    }
    public void init(){
        logger.info("init");
    }
    public void destroy(){
        logger.info("destroy");
    }
 
    /**
     * 客户数据维护
     * 维护auth.torganization,customer.trorganization,customer.tsorganization数据
     * 目标：定期扫描customer.tsorganization数据,维护到auth.torganization和trorganization,customer中
     * 解决：多种客户来源方式，包括但不限于客户导入，客户新增，微信+网厅新增关联企业过程微服务调用链路某个节点异常导致的数据缺失问题
     */
    private void syncTsorganizationTrorganizationTorganizationIndb() throws ExecutionException, InterruptedException {
        List<TSOrganizationRecord> records = DBvisitor.dsl.selectFrom(T_S_ORGANIZATION).where(T_S_ORGANIZATION.TYPE.eq(OrgType.DLYH.getValue())).fetchInto(TSOrganizationRecord.class);
        List<TROrganizationRecord> mappings = DBvisitor.dsl.selectFrom(T_R_ORGANIZATION).fetchInto(TROrganizationRecord.class);
        List<record> customersInAuth = (List<record>) Rest.get(ServiceUrl.AUTH.concat("/organizations/").concat(OrgType.DLYH.getValue()+"/").concat("all"), List.class).get().stream().map(x -> new record((String)(((Map) x).get("id")),(String)(((Map) x).get("name")))).collect(Collectors.toList());
//        List<record> customersInAuth = (List<record>) Rest.get("http://localhost:8082/authentication/organizations/256/all", List.class).get().stream().map(x -> {
//            return new record((String)(((Map) x).get("id")),(String)(((Map) x).get("name")));
//        }).collect(Collectors.toList());
        List<String> tsIdMappings = mappings.stream().map(TROrganizationRecord::getOrganizationId).distinct().collect(Collectors.toList());
        List<TROrganizationRecord> missingMappings = new CopyOnWriteArrayList<>(); //映射缺省的数据
        List<record> missingTorgnizations = new CopyOnWriteArrayList<>(); //auth库缺省的数据
        List<String> customersName = customersInAuth.stream().map(record::getValue).distinct().collect(Collectors.toList());
        records.stream().forEach(record -> {
            String sId = record.getId();
            String name = record.getName();
            if (!customersName.contains(name)) { //auth 的客户库里没数据
                String uuid = UUIDUtil.getUUID();
                missingTorgnizations.add(new record(sId, uuid));
                if (!tsIdMappings.contains(sId)) { //且customer无映射关系
                    missingMappings.add(buildTrorganization(record, uuid));
                }
            } else { // auth 的客户库里有数据
                if (!tsIdMappings.contains(sId)) { //但customer无映射关系
                    record data = customersInAuth.get(customersName.indexOf(name));
                    missingMappings.add(buildTrorganization(record, data.getKey()));
                }
            }
        });
        String authSql = getSql(records,missingTorgnizations);
        if (!authSql.endsWith("VALUES;")) {
            logger.info("要操作的auth库sql："+authSql);
            int row = DBvisitor.jdbcTemplate.update(authSql);
            logger.info("auth库插入了{}条数据",row);
        }
        if (!missingMappings.isEmpty()) {
            logger.info("要维护的customer.trorganiztion库的数据个数："+missingMappings.size());
            StringBuilder stringBuilder = new StringBuilder();
            stringBuilder.append("INSERT INTO `customer_tuiguang`.`t_r_organization`(`id`, `name`,`type`,`organization_id`,`create_time`,`last_modified_time`) VALUES (");
            missingMappings.stream().forEach(x-> {
                String createtime = Objects.nonNull(x.getCreateTime())?x.getCreateTime().toString():new Timestamp(new Date().getTime()).toString();
                String modifiedtime = Objects.nonNull(x.getLastModifiedTime())?x.getLastModifiedTime().toString():new Timestamp(new Date().getTime()).toString();
                String name = x.getName() == null ? "" : x.getName();
                stringBuilder.append("'"+x.getId()+"',").append("'"+name+"',").append("'256',").append("'"+x.getOrganizationId()+"',").append("'"+ createtime +"',").append("'"+ modifiedtime +"'),(");
            });
            String mappingSql = stringBuilder.toString();
            mappingSql = mappingSql.substring(0,mappingSql.length()-2);
            int mappingInsert = DBvisitor.jdbcTemplate.update(mappingSql);
//            int length = dsl.batchInsert(missingMappings).execute().length; 插入太耗时
//            logger.info("customer.trorganiztion插入{}条数据",length);
            logger.info("customer.trorganiztion插入{}条数据",mappingInsert);
        }
    }
 
    private TROrganizationRecord buildTrorganization(TSOrganizationRecord record,String tId) {
        TROrganizationRecord organizationRecord = new TROrganizationRecord();
        BeanUtils.copyProperties(record,organizationRecord);
        organizationRecord.setId(tId);
        organizationRecord.setOrganizationId(record.getId());
        organizationRecord.setPkId(null);
        return organizationRecord;
    }
 
    private String getSql(List<TSOrganizationRecord> records,List<record> missingTorgnizations){
        StringBuilder sqlbuilder = new StringBuilder();
        /**
         * INSERT INTO `authentication_tuiguang`.`t_organization`(`pk`, `id`, `name`, `alias`, `status`, `type`, `parent_id`, `create_time`, `last_modified_time`, `province_code`, `city_code`, `county_code`, `audit_status`, `applicant`, `audit_reason`, `order`, `power_source`) VALUES (490, '7830A3EC-9610-4260-B29D-35935FA23613-00820', '山东新和成维生素有限公司', NULL, 1, 256, NULL, '2019-01-11 20:16:34', '2019-01-11 20:16:34', NULL, NULL, NULL, NULL, NULL, NULL, 0, NULL);
         */
        List<String> keys = missingTorgnizations.stream().map(record::getKey).collect(Collectors.toList());
        List<String> values = missingTorgnizations.stream().map(record::getValue).collect(Collectors.toList());
        sqlbuilder.append("INSERT INTO `authentication_tuiguang`.`t_organization`(`id`, `name`, `status`, `type`, `create_time`, `last_modified_time`, `order`) VALUES (");
        records.stream().filter(x -> keys.contains(x.getId())).forEach(s -> {
            try {
                sqlbuilder.append(Deal(values.get(keys.indexOf(s.getId())))).append(",")
                        .append(Deal(s.getName())).append(",")
                        .append(1).append(",")
                        .append(OrgType.DLYH.getValue()).append(",")
                        .append(Deal(s.getCreateTime().toString())).append(",")
                        .append(Deal(s.getLastModifiedTime().toString())).append(",")
                        .append(0).append("),(");
            } catch (InstantiationException e) {
                e.printStackTrace();
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }
        });
        String sql = sqlbuilder.toString();
        sql = sql.substring(0, sql.length()-2).concat(";");
        return sql;
    }
 
    private Object Deal(Object obj) throws InstantiationException, IllegalAccessException {
        if (obj == null) {
            if ((obj instanceof String)) {
                return "''";
            } else {
                return null;
            }
        } else {
            if ((obj instanceof String)) {
                return "'"+ obj.toString()  +"'";
            } else {
                return obj.toString();
            }
        }
    }
 
    @Data
    @AllArgsConstructor
    public  class record {
        private String key;
        private String value;
    }
 
}