s3 upload files with shell script

TL; DR

#!/usr/bin/env bash
# hangj.cnblogs.com


s3_key="Q3AM3UQ867SPQQA43P2F"
s3_secret="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
session_token="Security-Token"

bucket="bucket-log"
tar_file="hello.log"


host="play.min.io"
region="cn-default"
service_name="s3"


method="PUT"
path="/${bucket}/${tar_file}"
query_string=""
content_type="application/octet-stream"
content_sha256="UNSIGNED-PAYLOAD"

amz_date=`date -u +%Y%m%dT%H%M%SZ`
signer_date=`echo ${amz_date} | cut -c1-8`
aws4_request="aws4_request"
scope="$signer_date/cn-default/s3/$aws4_request"

content_length=`wc -c $tar_file | awk '{print $1}'`
content_md5=`openssl dgst -md5 -binary $tar_file | base64`

date_key=`echo -en ${signer_date} | openssl dgst -sha256 -binary -hmac "AWS4${s3_secret}"`
date_region_key=`echo -en $region | openssl dgst -sha256 -binary -hmac "${date_key}"`
date_region_service_key=`echo -en $service_name | openssl dgst -sha256 -binary -hmac "${date_region_key}"`
signing_key=`echo -en $aws4_request | openssl dgst -sha256 -binary -hmac "${date_region_service_key}"`


headers="content-length:$content_length\ncontent-md5:$content_md5\ncontent-type:$content_type\nhost:$host\nx-amz-content-sha256:$content_sha256\nx-amz-date:$amz_date\nx-amz-security-token:$session_token"
signed_headers="content-length;content-md5;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"
canonical_request="$method\n$path\n$query_string\n$headers\n\n$signed_headers\n$content_sha256"
canonical_request_hash=`echo -en "$canonical_request" |openssl sha256`
string_to_sign="AWS4-HMAC-SHA256\n$amz_date\n$scope\n$canonical_request_hash"

printf "string_to_sign: $string_to_sign"

signature=`echo -en "${string_to_sign}" | openssl dgst -sha256 -hmac "${signing_key}"`

echo
echo $signature


curl -v -X ${method} -T "${tar_file}" \
          -H "Host: $host" \
          -H "Accept-Encoding: identity" \
          -H "Content-Length: ${content_length}" \
          -H "Content-MD5: ${content_md5}" \
          -H "x-amz-content-sha256: ${content_sha256}" \
          -H "X-Amz-Security-Token: ${session_token}" \
          -H "x-amz-date: ${amz_date}" \
          -H "Content-Type: ${content_type}" \
          -H "Authorization: AWS4-HMAC-SHA256 Credential=${s3_key}/${scope}, SignedHeaders=$signed_headers, Signature=${signature}" \
          https://$host${path}

缘起

想给自己的 Android app 加一个上传日志的功能，找到了 Minio , 但是 Android Studio 上死活用不了

可是它不就是一个 HTTP 的 PUT 请求嘛，只不过需要拼接一些参数和签名，我用 okhttp 搞一下就行了。但是我对 java 不熟，所以写个 python 版 代码研究一下

region = "cn-default"
s3_key = "Q3AM3UQ867SPQQA43P2F"
s3_secret = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
session_token = "Security-Token"

client = Minio(
    "play.min.io",
    access_key = s3_key,
    secret_key = s3_secret,
    session_token = session_token,
    # credentials = credentials,
    region = region,
    secure = True,

    # 这个是为了抓包，方便后面分析请求格式
    http_client = urllib3.ProxyManager(
        "http://localhost:8080/",
        cert_reqs = "CERT_REQUIRED",
        ca_certs = '/Users/hangj/.mitmproxy/mitmproxy-ca-cert.pem',
    ),
)

client.fput_object(
    "bucket-log", "hello.log", "/Users/hangj/dev/tmp/hello.log",
)

最终抓包结合 minio-py 的 signer.py 终于弄清楚协议细节, 手动拼接 curl 请求成功

接下来拼接 okhttp 也就不是什么问题了

参考链接

Upload to minio via curl
Amazon S3 REST API with curl