apache ozone配置(包含Kerberos认证)

环境说明

  1. 采用docker + compose方式运行
  2. Ozone镜像选择apache/ozone:1.4.0
  3. kerberos部署于宿主机上（Rocky Linux 9.4）
  4. 宿主机主机名：ozone.example.com，ip：192.168.121.128

部署配置

Kerberos部署

Kerberos配置记录

Kerberos配置

  • /etc/krb5.conf
# /etc/krb5.conf
includedir /etc/krb5.conf.d/

[logging]
    default = FILE:/var/log/krb5libs.log
    kdc = FILE:/var/log/krb5kdc.log
    admin_server = FILE:/var/log/kadmind.log

[libdefaults]
    dns_lookup_realm = false
    dns_lookup_kdc = false
    dns_canonicalize_hostname = false
    rdns = false
    ticket_lifetime = 24h
    renew_lifetime = 7d
    forwardable = true
    pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt
    spake_preauth_groups = edwards25519
    qualify_shortname = ""
    default_realm = EXAMPLE.COM
    # default_ccache_name = KEYRING:persistent:%{uid}

[realms]
EXAMPLE.COM = {
    kdc = ozone.example.com
    admin_server = ozone.example.com
}

[domain_realm]
.example.com = EXAMPLE.COM
example.com = EXAMPLE.COM
  • /var/kerberos/krb5kdc/kdc.conf
# /var/kerberos/krb5kdc/kdc.conf
[kdcdefaults]
    kdc_ports = 88
    kdc_tcp_ports = 88
    spake_preauth_kdc_challenge = edwards25519

[realms]
EXAMPLE.COM = {
     master_key_type = aes256-cts-hmac-sha384-192
     acl_file = /var/kerberos/krb5kdc/kadm5.acl
     dict_file = /usr/share/dict/words
     default_principal_flags = +preauth
     admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
     supported_enctypes = aes256-cts-hmac-sha384-192:normal aes128-cts-hmac-sha256-128:normal aes256-cts-hmac-sha1-96:normal aes128-cts-hmac-sha1-96:normal camellia256-cts-cmac:normal camellia128-cts-cmac:normal arcfour-hmac-md5:normal
     # Supported encryption types for FIPS mode:
     #supported_enctypes = aes256-cts-hmac-sha384-192:normal aes128-cts-hmac-sha256-128:normal
}
  • 创建addprinc和keytab
# 初始化数据库
kdb5_util create -s -P ozone@example.com

# 添加管理员
kadmin.local addprinc -pw admin@example admin/admin@EXAMPLE.COM

# 创建ozone服务使用的princ
kadmin.local -q "addprinc -randkey scm/scm@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey om/om@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey s3g/s3g@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey recon/recon@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey dn/dn@EXAMPLE.COM"

# 生成princ对应的keytab文件
kadmin.local -q "ktadd -k /etc/security/keytabs/ozone.keytab scm/scm@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/ozone.keytab om/om@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/ozone.keytab s3g/s3g@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/ozone.keytab recon/recon@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/ozone.keytab dn/dn@EXAMPLE.COM"

# 为ozone中http服务生成princ
kadmin.local -q "addprinc -randkey HTTP/scm@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey HTTP/om@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey HTTP/s3g@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey HTTP/recon@EXAMPLE.COM"
kadmin.local -q "addprinc -randkey HTTP/dn@EXAMPLE.COM"

# 生成keytab文件，注意必须命名为HTTP.keytab（官方文档和测试中验证）
kadmin.local -q "ktadd -k /etc/security/keytabs/HTTP.keytab HTTP/scm@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/HTTP.keytab HTTP/om@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/HTTP.keytab HTTP/s3g@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/HTTP.keytab HTTP/recon@EXAMPLE.COM"
kadmin.local -q "ktadd -k /etc/security/keytabs/HTTP.keytab HTTP/dn@EXAMPLE.COM"
  • 启动服务
systemctl start krb5kdc kadmin

部署Docker + Compose （略）

Ozone部署配置

services:
   dn:		# datanode服务
      image: apache/ozone:1.4.0
      hostname: dn				# 自定义主机名
      deploy:
        replicas: 3				# 设置datanode为3副本（节点）
      ports:
         - 9864
      command: ["ozone","datanode"]
      volumes:
        - /etc/security/keytabs:/etc/security/keytabs:ro	# keytab文件
        - /etc/krb5.conf:/etc/krb5.conf:ro								# kerberos配置文件
        - ./ozone-site.xml:/opt/hadoop/etc/hadoop/ozone-site.xml:ro		# ozone核心配置
        - ./core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro			# 客户端关于kerberos配置（重要）
      extra_hosts:
        - "ozone.example.com:192.168.121.128"	  # 注入kdc域名解析到/etc/hosts
   om:
      image: apache/ozone:1.4.0
      hostname: om
      ports:
         - 9874:9874
      environment:
         ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION  # om服务启动前需要初始化
         WAITFOR: scm:9876    # 依赖scm服务先启动
      command: ["ozone","om"]
      volumes:
        - /etc/security/keytabs:/etc/security/keytabs:ro
        - /etc/krb5.conf:/etc/krb5.conf:ro
        - ./ozone-site.xml:/opt/hadoop/etc/hadoop/ozone-site.xml:ro
        - ./core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      extra_hosts:
        - "ozone.example.com:192.168.121.128"
   scm:
      image: apache/ozone:1.4.0
      hostname: scm
      ports:
         - 9876:9876
      environment:
         ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
      command: ["ozone","scm"]
      volumes:
        - /etc/security/keytabs:/etc/security/keytabs:ro
        - /etc/krb5.conf:/etc/krb5.conf:ro
        - ./ozone-site.xml:/opt/hadoop/etc/hadoop/ozone-site.xml:ro
        - ./core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      extra_hosts:
        - "ozone.example.com:192.168.121.128"
   recon:
      image: apache/ozone:1.4.0
      hostname: recon
      ports:
         - 9888:9888
      command: ["ozone","recon"]
      volumes:
        - /etc/security/keytabs:/etc/security/keytabs:ro
        - /etc/krb5.conf:/etc/krb5.conf:ro
        - ./ozone-site.xml:/opt/hadoop/etc/hadoop/ozone-site.xml:ro
        - ./core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      extra_hosts:
        - "ozone.example.com:192.168.121.128"
   s3g:
      image: apache/ozone:1.4.0
      hostname: s3g
      ports:
         - 9878:9878
      command: ["ozone","s3g"]
      volumes:
        - /etc/security/keytabs:/etc/security/keytabs:ro
        - /etc/krb5.conf:/etc/krb5.conf:ro
        - ./ozone-site.xml:/opt/hadoop/etc/hadoop/ozone-site.xml:ro
        - ./core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      extra_hosts:
        - "ozone.example.com:192.168.121.128"

Ozone核心配置

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <!-- Ozone Manager (OM) Configuration -->
    <property>
        <name>ozone.om.address</name>
        <value>om</value>
    </property>
    <property>
        <name>ozone.om.http-address</name>
        <value>om:9874</value>
    </property>

    <!-- Storage Container Manager (SCM) Configuration -->
    <property>
        <name>ozone.scm.names</name>
        <value>scm</value>
    </property>
    <property>
        <name>ozone.scm.datanode.id.dir</name>
        <value>/data</value>
    </property>
    <property>
        <name>ozone.scm.block.client.address</name>
        <value>scm</value>
    </property>
    <property>
        <name>ozone.metadata.dirs</name>
        <value>/data/metadata</value>
    </property>
    <property>
        <name>ozone.scm.client.address</name>
        <value>scm</value>
    </property>

    <!-- Replication Factor -->
    <property>
        <name>ozone.replication</name>
        <value>1</value>
    </property>

    <!-- Datanode (DN) Configuration -->
    <property>
        <name>hdds.datanode.dir</name>
        <value>/data/hdds</value>
    </property>

    <!-- Recon Configuration -->
    <property>
        <name>ozone.recon.db.dir</name>
        <value>/data/metadata/recon</value>
    </property>
    <property>
        <name>ozone.recon.address</name>
        <value>recon:9891</value>
    </property>

    <!-- Security Configuration -->
    <property>
        <name>ozone.security.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>ozone.security.http.kerberos.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>ozone.http.filter.initializers</name>
        <value>org.apache.hadoop.security.AuthenticationFilterInitializer</value>
    </property>
    <property>
        <name>hadoop.security.authentication</name>
        <value>KERBEROS</value>
    </property>

    <!-- Kerberos Principals and Keytab Files -->
    <property>
        <name>hdds.scm.kerberos.principal</name>
        <value>scm/scm@EXAMPLE.COM</value>
    </property>
    <property>
        <name>hdds.scm.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/ozone.keytab</value>
    </property>
        <property>
        <name>hdds.scm.http.auth.type</name>
        <value>kerberos</value>
    </property>
    <property>
        <name>hdds.scm.http.auth.kerberos.principal</name>
        <value>HTTP/scm@EXAMPLE.COM</value>
    </property>
    <property>
        <name>hdds.scm.http.auth.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/HTTP.keytab</value>
    </property>
    <property>
        <name>ozone.om.kerberos.principal</name>
        <value>om/om@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.om.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/ozone.keytab</value>
    </property>
        <property>
        <name>ozone.om.http.auth.type</name>
        <value>kerberos</value>
    </property>
    <property>
        <name>ozone.om.http.auth.kerberos.principal</name>
        <value>HTTP/om@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.om.http.auth.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/HTTP.keytab</value>
    </property>
    <property>
        <name>ozone.s3g.kerberos.principal</name>
        <value>s3g/s3g@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.s3g.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/ozone.keytab</value>
    </property>
        <property>
        <name>ozone.s3g.http.auth.type</name>
        <value>kerberos</value>
    </property>
    <property>
        <name>ozone.s3g.http.auth.kerberos.principal</name>
        <value>HTTP/s3g@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.s3g.http.auth.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/HTTP.keytab</value>
    </property>
    <property>
        <name>dfs.datanode.kerberos.principal</name>
        <value>dn/dn@EXAMPLE.COM</value>
    </property>
    <property>
        <name>dfs.datanode.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/ozone.keytab</value>
    </property>
    <property>
        <name>hdds.datanode.http.auth.kerberos.principal</name>
        <value>HTTP/dn@EXAMPLE.COM</value>
    </property>
    <property>
        <name>hdds.datanode.http.auth.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/HTTP.keytab</value>
    </property>
    <property>
        <name>ozone.recon.kerberos.principal</name>
        <value>recon/recon@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.recon.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/ozone.keytab</value>
    </property>
        <property>
        <name>ozone.recon.http.auth.type</name>
        <value>kerberos</value>
    </property>
    <property>
        <name>ozone.recon.http.auth.kerberos.principal</name>
        <value>HTTP/recon@EXAMPLE.COM</value>
    </property>
    <property>
        <name>ozone.recon.http.auth.kerberos.keytab.file</name>
        <value>/etc/security/keytabs/HTTP.keytab</value>
    </property>
</configuration>

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>hadoop.security.authentication</name>
        <value>kerberos</value>
    </property>
</configuration>

启动服务

# 配置文件情况
[root@ozone ~]# cd ozone/
[root@ozone ozone]# ls
core-site.xml  docker-compose.yaml  ozone-site.xml
[root@ozone ozone]#

# 启动服务
[root@ozone ozone]# docker-compose up -d
[+] Running 8/8
 ✔ Network ozone_default    Created                                                    0.0s
 ✔ Container ozone-dn-3     Started                                                    0.5s
 ✔ Container ozone-om-1     Started                                                    0.4s
 ✔ Container ozone-scm-1    Started                                                    0.4s
 ✔ Container ozone-recon-1  Started                                                    0.5s
 ✔ Container ozone-s3g-1    Started                                                    0.4s
 ✔ Container ozone-dn-1     Started                                                    1.0s
 ✔ Container ozone-dn-2     Started                                                    0.7s
[root@ozone ozone]#

# 大概需要个1~3分钟启动（具体根据机器配置而定）
# 通过docker-compose logs -f监控日志变化，出现下面的日志基本就算启动好了
scm-1    | 2024-11-25 03:59:19 INFO  PipelineReportHandler:136 - Opened pipeline PipelineID=09781053-2bf1-4c18-b4ae-a1fc84ae0abe
scm-1    | 2024-11-25 03:59:19 INFO  PipelineReportHandler:136 - Opened pipeline PipelineID=75b3b02b-25f3-47dd-a860-b157e7940735
scm-1    | 2024-11-25 03:59:20 INFO  PipelineReportHandler:136 - Opened pipeline PipelineID=1ed2e5c9-6666-4959-86a5-1e9e73a405c0
scm-1    | 2024-11-25 03:59:24 INFO  PipelineReportHandler:136 - Opened pipeline PipelineID=3acf8c33-7042-44d9-82b3-d1d9b33a2f66
scm-1    | 2024-11-25 03:59:25 INFO  PipelineReportHandler:136 - Opened pipeline PipelineID=130e2f97-4bdd-4658-b4f3-12466b5a03e4
scm-1    | 2024-11-25 03:59:54 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-3.ozone_default:36667 / 172.18.0.6:36667
scm-1    | 2024-11-25 03:59:54 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-2.ozone_default:39369 / 172.18.0.7:39369
scm-1    | 2024-11-25 03:59:55 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-1.ozone_default:37525 / 172.18.0.8:37525
scm-1    | 2024-11-25 04:00:14 INFO  Server:2139 - Auth successful for recon/recon@EXAMPLE.COM (auth:KERBEROS) from ozone-recon-1.ozone_default:40197 / 172.18.0.5:40197
scm-1    | 2024-11-25 04:00:24 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-3.ozone_default:38465 / 172.18.0.6:38465
scm-1    | 2024-11-25 04:00:24 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-2.ozone_default:43553 / 172.18.0.7:43553
scm-1    | 2024-11-25 04:00:25 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-1.ozone_default:33435 / 172.18.0.8:33435
scm-1    | 2024-11-25 04:00:54 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-3.ozone_default:38233 / 172.18.0.6:38233
scm-1    | 2024-11-25 04:00:54 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-2.ozone_default:40507 / 172.18.0.7:40507
scm-1    | 2024-11-25 04:00:55 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-1.ozone_default:43885 / 172.18.0.8:43885
scm-1    | 2024-11-25 04:01:24 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-3.ozone_default:34651 / 172.18.0.6:34651
scm-1    | 2024-11-25 04:01:24 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-2.ozone_default:37087 / 172.18.0.7:37087
scm-1    | 2024-11-25 04:01:25 INFO  Server:2139 - Auth successful for dn/dn@EXAMPLE.COM (auth:KERBEROS) from ozone-dn-1.ozone_default:35009 / 172.18.0.8:35009

设置S3 Secret

由于启用了安全配置，通过S3访问必须使用Secret才行，下面通过命名获取s3g/s3g@EXAMPLE.COM对应的Secret：

[root@ozone ozone]# docker exec -it ozone-s3g-1 bash
bash-4.2$ kinit -kt /etc/security/keytabs/ozone.keytab s3g/s3g@EXAMPLE.COM
bash-4.2$ klist
Ticket cache: FILE:/tmp/krb5cc_1000
Default principal: s3g/s3g@EXAMPLE.COM

Valid starting     Expires            Service principal
11/25/24 04:05:42  11/26/24 04:05:42  krbtgt/EXAMPLE.COM@EXAMPLE.COM
        renew until 11/25/24 04:05:42
bash-4.2$ ozone s3 getsecret
2024-11-25 04:05:54 WARN  NativeCodeLoader:60 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-25 04:05:55 INFO  ClientTrustManager:148 - Loading certificates for client.
awsAccessKey=s3g/s3g@EXAMPLE.COM
awsSecret=297fbdeee8923e005f7acf63e8253eb04a013bf66f3660ff7c73e34d3129be2c
isDeleted=false
transactionLogIndex=0
bash-4.2$

S3客户端测试

aws-cli

# ozone镜像自带了aws-cli，通过s3g容器操作
[root@ozone ~]# docker exec -it ozone-s3g-1 bash

# 生成aws配置文件
bash-4.2$ aws configure
AWS Access Key ID [None]: s3g/s3g@EXAMPLE.COM
AWS Secret Access Key [None]: 297fbdeee8923e005f7acf63e8253eb04a013bf66f3660ff7c73e34d3129be2c
Default region name [None]:
Default output format [None]:
bash-4.2$ cat ~/.aws/credentials
[default]
aws_access_key_id = s3g/s3g@EXAMPLE.COM
aws_secret_access_key = 297fbdeee8923e005f7acf63e8253eb04a013bf66f3660ff7c73e34d3129be2c
bash-4.2$

# 创建bucket
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 mb s3://aws-cli
make_bucket: aws-cli

# 列出bucket
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 ls
2024-11-25 05:58:15 aws-cli

# 上传文件
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 cp Dockerfile s3://aws-cli/Dockerfile
upload: ./Dockerfile to s3://aws-cli/Dockerfile

# 查看bucket文件
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 ls s3://aws-cli
2024-11-25 06:02:22        877 Dockerfile

# 删除文件
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 rm s3://aws-cli/Dockerfile
delete: s3://aws-cli/Dockerfile

# 删除bucket
bash-4.2$ aws --endpoint-url http://ozone.example.com:9878 s3 rb s3://aws-cli
remove_bucket: aws-cli

S3Browser

posted @ 2024-12-02 15:05  longtds  阅读(21)  评论(0)  编辑  收藏  举报