sparkListener配合钉钉使用

自定义Listener来实现对spark任务的运行状态的监控：

package org.shydow.example

import org.apache.spark.internal.Logging
import org.apache.spark.scheduler.{JobResult, SparkListener, SparkListenerJobEnd, SparkListenerJobStart}
import org.shydow.util.ding.{DingTalkService, PushMessage}

/**
 * @author Shydow
 * @date 2021/12/18 10:13
 */

class DefaultListener extends SparkListener with Logging {

  override def onJobStart(jobStart: SparkListenerJobStart): Unit = {
    val id: Int = jobStart.jobId
    val timestamp: Long = jobStart.time
    val message = s"警告： Job id: $id, 时间：$timestamp, 开始执行===="
    pushMsg(message)
  }


  override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
    val id: Int = jobEnd.jobId
    val result: JobResult = jobEnd.jobResult
    val timestamp: Long = jobEnd.time
    val msg = s"警告：Job id：$id, 时间：$timestamp, 运行结果：$result===="
    pushMsg(msg)
  }


  private def pushMsg(msg: String) = {
    val hook = "https://oapi.dingtalk.com/robot/send?access_token=..."
    val message = new PushMessage
    message.setMsgType("text")
    message.setText(msg)
    message.setWebhook(hook)
    message.setIsAtAll(false)
    DingTalkService.sendMessage(message)
  }
}

package org.shydow.util.ding;

import com.dingtalk.api.DefaultDingTalkClient;
import com.dingtalk.api.request.OapiRobotSendRequest;
import com.dingtalk.api.response.OapiRobotSendResponse;
import com.taobao.api.ApiException;

/**
 * @author Shydow
 * @date 2021/12/18 10:33
 */

public class DingTalkService {

    private static final String TEXT = "text";
    private static final String LINK = "link";
    private static final String MARKDOWN = "markdown";

    public static void sendMessage(PushMessage in) throws ApiException {

        DefaultDingTalkClient dingTalkClient = new DefaultDingTalkClient(in.getWebhook());
        OapiRobotSendRequest request = new OapiRobotSendRequest();

        // 指定at的人
        OapiRobotSendRequest.At at = new OapiRobotSendRequest.At();
        if (in.getIsAtAll() || in.getMobileList() == null || in.getMobileList().size() == 0) {
            //推送所有人
            at.setIsAtAll(true);
        } else {
            //推送指定用户
            at.setAtMobiles(in.getMobileList());
            at.setIsAtAll(false);
        }
        request.setAt(at);

        // 发送文本
        if (TEXT.equals(in.getMsgType())) {
            request.setMsgtype(TEXT);
            OapiRobotSendRequest.Text text = new OapiRobotSendRequest.Text();
            text.setContent(in.getText());
            request.setText(text);
        }
        OapiRobotSendResponse res = dingTalkClient.execute(request);
    }

}

package org.shydow.util.ding;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.util.List;

/**
 * @author Shydow
 * @date 2021/12/18 10:27
 */

@Data
@NoArgsConstructor
@AllArgsConstructor
public class PushMessage {
    private String msgType;
    private String webhook;
    private String text;
    private List<String> mobileList;
    private Boolean isAtAll;
}

 

spark中注册监听器：

val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("Listener")
val sc = new SparkContext(conf)
sc.setLogLevel("WARN")
sc.addSparkListener(new DefaultListener) // 注册监听器