java读取linux服务器下某文档的内容

21.java读取linux服务器下某文档的内容2024-12-08

22.java实现从HDFS上下载文件及文件夹的功能，以流形式输出，便于用户自定义保存任何路径下2024-12-09

使用步骤：

使用步骤：

共3步：
第一步：添加依赖
第二步：配置文件
第三步：代码调用

依赖

<!--java读spark-->
 <dependency>
     <groupId>ch.ethz.ganymed</groupId>
     <artifactId>ganymed-ssh2</artifactId>
     <version>build210</version>
 </dependency>
 <!--java读spark-->
 <dependency>
     <groupId>com.jcraft</groupId>
     <artifactId>jsch</artifactId>
     <version>0.1.54</version>
 </dependency>

配置文件

# bigdata
usr=root
pwd=geecentos

代码

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Vector;

import ch.ethz.ssh2.Connection;
import ch.ethz.ssh2.SFTPv3Client;
import ch.ethz.ssh2.SFTPv3DirectoryEntry;
import ch.ethz.ssh2.Session;
import ch.ethz.ssh2.StreamGobbler;

import com.geespace.microservices.calculate.execute.engine.bean.pojo.SparkRecord;
import com.geespace.microservices.calculate.execute.engine.dao.SparkRecordMapper;
import com.geespace.microservices.calculate.execute.engine.response.JobSubmitResponse;
import com.geespace.microservices.calculate.execute.engine.response.Msg;
import com.geespace.microservices.calculate.execute.engine.response.Response;

import lombok.extern.slf4j.Slf4j;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.RestTemplate;

/**
     * login 注意：创建远程连接，默认连接端口为22，如果不使用默认，可以使用方法
     *
     * @param submissionId
     *            submissionId
     * @param ret
     *            ret
     * @author liudz
     * @date 2020/5/11
     * @return 执行结果
     **/
    public String login(String submissionId, ResponseEntity<JobSubmitResponse> ret) {
        String result = "";
        String ip = ret.getBody().getWorkerHostPort().substring(0, ret.getBody().getWorkerHostPort().indexOf(":"));
        Connection conn = null;
        Session ss = null;
        String directory = "/home/spark/work/" + submissionId;
        try {
            conn = new Connection(ip);
            conn.connect();
            boolean b = conn.authenticateWithPassword(usr, pwd);
            if (!b) {
                throw new IOException("Authentication failed.");
            } else {
                SFTPv3Client sft = new SFTPv3Client(conn);
                Vector<?> v = sft.ls(directory);
                for (int i = 0; i < v.size(); i++) {
                    SFTPv3DirectoryEntry s = new SFTPv3DirectoryEntry();
                    s = (SFTPv3DirectoryEntry) v.get(i);
                    if ("stdout".equals(s.filename)) {
                        ss = conn.openSession();
                        ss.execCommand("cat ".concat("/home/spark/work/" + submissionId + "/" + s.filename));
                        InputStream is = new StreamGobbler(ss.getStdout());
                        BufferedReader bs = new BufferedReader(new InputStreamReader(is));
                        while (true) {
                            String line = bs.readLine();
                            if (line == null) {
                                break;
                            } else {
                                result += line + "\n";
                            }
                        }
                        bs.close();
                    }
                }
                ss.close();
                conn.close();
            }
        } catch (IOException e) {
            log.error("用户%s密码%s登录服务器%s失败！", usr, pwd, ip, "--ERROR--e：" + e.getMessage());
        }
        System.out.print(result);
        return result;
    }

JobSubmitResponse

package com.geespace.microservices.calculate.execute.engine.response;

import lombok.Data;

/**
 * @author: liudz
 * @date: 2020-04-27
 */
@Data
public class JobSubmitResponse {
    /**
     * 动作名称
     */
    private String action;
    /**
     * 信息
     */
    private String message;
    /**
     * spark版本
     */
    private String serverSparkVersion;
    /**
     * 提交id
     */
    private String submissionId;
    /**
     * 成功与否状态
     */
    private String success;
    /**
     * driver状态
     */
    private String driverState;
    /**
     * workerHostPort
     */
    private String workerHostPort;
    /**
     * workerId
     */
    private String workerId;
}