一、做好镜像文件
1、编辑脚本 pull.sh
#!/bin/bash if [ $# -ne 1 ];then echo "Usage:$0 filename" exit 1 fi file=$1 if [ ! -f $file ];then echo "the $file is not a file" exit 2 fi count=0 while read line #使用read命令循环读取文件内容，并将读取的文件内容赋值给变量line do line1=${line:16} line2=${line1/\//_} line2=${line2/:/_} let count++ echo "now pull $count: $line " >> replicat.log docker pull $line >/dev/null if [ $? == 0 ];then echo "now tag $count: $line " >> replicat.log zyd_tag=${line/harbor/harbor-zyd1} docker tag $line $zyd_tag docker save -o $line2.tar $zyd_tag >/dev/null echo "now gzip $count: $line2" >> replicat.log echo "now tar " tar -czvf $line2.tar.gz $line2.tar >/dev/null rm -f $line2.tar echo "$count: $line successed" >> success.log docker rmi $line > /dev/null docker rmi $zyd_tag > /dev/null else echo "pull failed $count: $line" >> failed.log fi done <$file #“done <$file”将整个while循环的标准输入指向文件$file echo -e "\ntotle $count lines read" >> replicat.log exit 0
拉取的镜像列表
[root@ops0002 zyd1230]# cat list3
harbor-test1.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005
执行操作 nohup sh pull.sh list3 &
执行过程如下
读取一个文件列表，按行执行操作 过程如下 docker pull harbor-test.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 拉取镜像 docker tag harbor-test.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 harbor-test2.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 镜像打标记 docker save -o enos_dataexplorer_tag_dataexplore_20191227_005.tar harbor-test2.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 将镜像保存为 gz文件 tar -czvf enos_dataexplorer_tag_dataexplore_20191227_005.tar.gz enos_dataexplorer_tag_dataexplore_20191227_005.tar 将镜像文件压缩为 tar.gz文件 rm -f enos_dataexplorer_tag_dataexplore_20191227_005.tarr 删除文件 docker rmi harbor-test.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 删除拉取的镜像 docker rmi harbor-test2.eniot.io/enos/dataexplorer:tag_dataexplore_20191227_005 删除打标机的镜像 -------------------------------- 最后留下的文件为 enos_dataexplorer_tag_dataexplore_20191227_005.tar.gz
2、下载文件
nginx配置下载
server { listen 8080; server_name localhost; autoindex on; autoindex_exact_size on; autoindex_localtime on; charset utf-8,gbk; #charset koi8-r; #access_log /var/log/nginx/host.access.log main; location / { root /data/images; #index index.html index.htm; } #error_page 404 /404.html; # redirect server error pages to the static page /50x.html # error_page 500 502 503 504 /50x.html; location = /50x.html { root /usr/share/nginx/html; } # proxy the PHP scripts to Apache listening on 127.0.0.1:80 # #location ~ \.php$ { # proxy_pass http://127.0.0.1; #} # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 # #location ~ \.php$ { # root html; # fastcgi_pass 127.0.0.1:9000; # fastcgi_index index.php; # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; # include fastcgi_params; #} # deny access to .htaccess files, if Apache's document root # concurs with nginx's one # #location ~ /\.ht { # deny all; #} }
3）以上功能使用 python脚本实现。
[root@harbor0001 images]# cat pull_image.py #!/usr/bin/env python # -*- coding: UTF-8 -*- import os,sys class MyError(BaseException): def __init__(self,msg): super().__init__() self.msg=msg def __str__(self): return '<%s>' %self.msg class Images: def cmd(self,comand): ret = os.system(comand) if ret != 0: with open('failure.txt', mode='a') as f: f.write(comand) f.write('\n') raise MyError('command error') def pull(self,image): '''拉取镜像''' comand = "docker pull %s" % image self.cmd(comand) def tag(self,source_image,desc_image): '''标记镜像''' comand = "docker tag %s %s" %(source_image,desc_image) self.cmd(comand) def save(self,image_tar,desc_image,image_tar_gz): '''保存镜像''' comand = "docker save -o %s %s" %(image_tar,desc_image) self.cmd(comand) comand2 = "tar -czvf %s %s" %(image_tar_gz,image_tar) self.cmd(comand2) comand3 = "rm -f %s" % image_tar self.cmd(comand3) def delete_image(self,source_image,desc_image): comand1 = "docker rmi %s" %source_image comand2 = "docker rmi %s" %desc_image self.cmd(comand1) self.cmd(comand2) if len(sys.argv) == 2: file = sys.argv[1] image = Images() with open(file,mode='r') as f: for line in f: image_error = line.split('/') image_error[-1] = image_error[-1].split('\n')[0] source_image = '/'.join(image_error) image_error[0] = 'harbor-test2.eniot.io' desc_image = '/'.join(image_error) image_tar = '_'.join(image_error) image_tar = image_tar.split(':') image_tar = '_'.join(image_tar) + ".tar" image_tar_gz = image_tar + ".gz" # print(image_tar) # print(image_tar_gz) # print(source_image) # print(desc_image) image.pull(source_image) image.tag(source_image, desc_image) image.save(image_tar, desc_image, image_tar_gz) image.delete_image(source_image, desc_image) with open('success.txt', mode='a') as f: f.write(line) else: print('input image file')
nohup python pull_image.py list &
二、推送镜像1
[root@harbor0001 ~]# cat a.sh #!/bin/bash docker pull harbor-test.eniot.io/${1} docker tag harbor-test.eniot.io/${1} harbor-test1.eniot.io/${1} docker push harbor-test1.eniot.io/${1}
三、推送镜像2
[root@harbor0001 images]# cat pull.sh #!/bin/bash if [ $# -ne 1 ];then echo "Usage:$0 filename" exit 1 fi file=$1 if [ ! -f $file ];then echo "the $file is not a file" exit 2 fi count=0 while read line #使用read命令循环读取文件内容，并将读取的文件内容赋值给变量line do let count++ echo "now pull $count: $line " >> replicat.log docker pull $line >/dev/null if [ $? == 0 ];then echo "now tag $count: $line " >> replicat.log india_tag=${line/cn2/india1} docker tag $line $india_tag echo " now push $india_tag" >> replicat.log docker push $india_tag if [ $? == 0 ]; then echo "------$count: $line successed" >> success.log echo " push $india_tag successed" >> replicat.log docker rmi $line > /dev/null docker rmi $india_tag > /dev/null else echo " push $india_tag failed" >> replicat.log echo " $count : push $india_tag failed" >> failed.log fi else echo "pull failed $count: $line" >> failed.log fi done <$file #“done <$file”将整个while循环的标准输入指向文件$file echo -e "\ntotle $count lines read" >> replicat.log exit 0
2）该脚本内容用python已经改写
[root@harbor0001 files]# cat push_image.py #!/usr/bin/env python # -*- coding: UTF-8 -*- import os,sys class MyError(BaseException): def __init__(self,msg): super().__init__() self.msg=msg def __str__(self): return '<%s>' %self.msg class Images: def cmd(self,comand): ret = os.system(comand) if ret != 0: with open('failure.txt', mode='a') as f: f.write(comand) f.write('\n') raise MyError('command error') def load(self,image_tar_gz,image): '''加载镜像''' image_tar = image_tar_gz.split(".gz")[0] comand1 = "tar -xf %s" %(image_tar_gz) self.cmd(comand1) comand2 = "rm -f %s" % image_tar_gz self.cmd(comand2) comand3 = "docker load -i %s" % image_tar self.cmd(comand3) comand4 = "docker push %s" % image self.cmd(comand4) comand5 = "docker rmi %s" % image self.cmd(comand5) comand6 = "rm -f %s" % image_tar self.cmd(comand6) def main(self): list_files = os.listdir('.') for image_tar_gz in list_files: if image_tar_gz.endswith('tar.gz'): new_image = image_tar_gz.split(".tar.gz")[0].split("_") if len(new_image) == 4: image = new_image[0] + '/' + new_image[1] + "/" + new_image[2] + ':' + new_image[3] self.load(image_tar_gz,image) with open('success.txt', mode='a') as f: f.write(image) f.write("\n") if __name__ == '__main__': image = Images() image.main()
注意上面镜像 bug 。标记的镜像 不能包含 “_”。以及下载的时候，就已经做了标记
三、镜像标记为： tag_nginx_20200227_005 ，有且只有 一个 tag 时。推送时，再给镜像地址
下载镜像
#!/usr/bin/env python # -*- coding: UTF-8 -*- import os,sys class MyError(BaseException): def __init__(self,msg): super().__init__() self.msg=msg def __str__(self): return '<%s>' %self.msg class Images: def cmd(self,comand): ret = os.system(comand) if ret != 0: with open('failure.txt', mode='a') as f: f.write(comand) f.write('\n') raise MyError('command error') def pull(self,image): '''拉取镜像''' comand = "docker pull %s" % image self.cmd(comand) def save(self,image_tar,desc_image,image_tar_gz): '''保存镜像''' comand = "docker save -o %s %s" %(image_tar,desc_image) self.cmd(comand) comand2 = "tar -czvf %s %s" %(image_tar_gz,image_tar) self.cmd(comand2) comand3 = "rm -f %s" % image_tar self.cmd(comand3) def delete_image(self,source_image): comand = "docker rmi %s" %source_image self.cmd(comand) if len(sys.argv) == 2: file = sys.argv[1] image = Images() with open(file,mode='r') as f: for line in f: image_error = line.split('/') if image_error[0] != 'harbor-test1.eniot.io': continue image_error[-1] = image_error[-1].split('\n')[0] source_image = '/'.join(image_error) desc_image = '/'.join(image_error) image_tar = '_'.join(image_error) image_tar = image_tar.split(':') image_tar = '_'.join(image_tar) + ".tar" image_tar_gz = image_tar + ".gz" image.pull(source_image) image.save(image_tar,source_image,image_tar_gz) image.delete_image(source_image) with open('success.txt', mode='a') as f: f.write(line) else: print('input image file')
推送镜像
#!/usr/bin/env python # -*- coding: UTF-8 -*- import os,sys,time class MyError(BaseException): def __init__(self,msg): super().__init__() self.msg=msg def __str__(self): return '<%s>' %self.msg class Images: def cmd(self,comand): ret = os.system(comand) if ret != 0: with open('failure.txt', mode='a') as f: f.write(comand) f.write('\n') raise MyError('command error') def load(self,image_tar_gz,source_image,desc_image): '''加载镜像''' image_tar = image_tar_gz.split(".gz")[0] comand1 = "tar -xf %s" %(image_tar_gz) self.cmd(comand1) comand2 = "rm -f %s" % image_tar_gz self.cmd(comand2) comand3 = "docker load -i %s" % image_tar self.cmd(comand3) comand4 = "docker tag %s %s" %(source_image,desc_image) self.cmd(comand4) comand5 = "docker push %s" % desc_image self.cmd(comand5) comand6 = "docker rmi %s" % source_image comand7 = "docker rmi %s" % desc_image self.cmd(comand6) self.cmd(comand7) comand8 = "rm -f %s" % image_tar self.cmd(comand8) def main(self): list_files = os.listdir('.') for image_tar_gz in list_files: if image_tar_gz.endswith('tar.gz'): new_image = image_tar_gz.split(".tar.gz")[0] if new_image.count("tag") == 1: new_image = new_image.split("_tag_") harbor_adress = "harbor-test2.eniot.io" app = new_image[0].split("_") source_image = app[0] + "/" + app[1] + "/" + app[2] + ":tag_" + new_image[1] desc_image = harbor_adress + "/" + app[1] + "/" + app[2] + ":tag_" + new_image[1] self.load(image_tar_gz,source_image,desc_image) with open('success.txt', mode='a') as f: f.write(desc_image) f.write("\n") if __name__ == '__main__': image = Images() image.main()
四、harbor里charts的详细信息
import requests class HarborApi(): def __init__(self, url): self.url = url self.session = requests.Session() self.api = self.url + '/api' # "chartrepo/{repo}/charts/{name}/{version}" def chartrepo(self): url = "/chartrepo/myproject/charts/nginx-app-orchestration/2.1.8-20200924001-tag" charturl = self.api + url chartrepo_info = self.session.get(charturl).json() print(chartrepo_info) if __name__ == "__main__": harbor_url = "https://harbor-test.eniot.io" res = HarborApi(harbor_url) res.chartrepo()
五、通过helm过滤镜像
helm template ${CHART_TAR} | grep -v "^[[:space:]]*#" | grep image: | grep -o -E "/[a-z0-9]([-/a-z0-9]*[a-z0-9])?:[a-zA-Z0-9]([-._a-zA-Z0-9]*[a-zA-Z0-9])?" | sed 's@^/@@g' | grep -v "^op/lion:" | grep -v "^op/lion-init:" | grep -v "^op/db-init:"| sort | uniq
1）通过charts获取images
#!/usr/bin/env python # -*- coding: UTF-8 -*- import wget import subprocess import openpyxl import os class HelmApi(): def __init__(self, url): self.url = url def charts_info(self): wb = openpyxl.load_workbook('chart_info.xlsx') sh = wb['Sheet1'] for cases in list(sh.rows)[1:]: project = cases[1].value service = cases[2].value version = cases[3].value self.chartrepo(project, service,version) def chartrepo(self,project,service,version): file_url = self.url + "/chartrepo/{project}/charts/{service}-{version}.tgz".format( project=project, service=service, version=version, ) # /root/test/{service}-{version}.tgz target_file = "{service}-{version}.tgz".format( service=service, version=version, ) try: wget.download(file_url, out=target_file) cmd = "helm template {target_file} | grep -v '^[[:space:]]*#' | grep image: | grep -o -E '/[a-z0-9]([-/a-z0-9]*[a-z0-9])?:[a-zA-Z0-9]([-._a-zA-Z0-9]*[a-zA-Z0-9])?' | sed 's@^/@@g' | grep -v '^op/lion:' | grep -v '^op/lion-init:' | grep -v '^op/db-init:'| sort | uniq".format( target_file=target_file, ) obj=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE) res = obj.stdout.read().decode('utf-8') with open('image.txt', mode='a', encoding='utf-8') as f: f.write(res) except Exception as e: with open('error_url.txt', mode='a', encoding='utf-8') as f: f.write(file_url) os.remove(target_file) if __name__ == "__main__": with open('image.txt', mode='w', encoding='utf-8') as f: f.write('') harbor_url = "https://harbor-test.eniot.io" res = HelmApi(harbor_url) res.charts_info()
2）体现chart和images的关系
#!/usr/bin/env python # -*- coding: UTF-8 -*- import wget import subprocess import openpyxl import os import json class HelmApi(): def __init__(self, url): self.url = url self.charts_list = [] def charts_info(self): wb = openpyxl.load_workbook('chart_info.xlsx') sh = wb['Sheet1'] for cases in list(sh.rows)[1:]: charts_namespace = cases[1].value charts_name = cases[2].value charts_version = cases[3].value self.chartrepo(charts_namespace, charts_name,charts_version) self.write_file(self.charts_list) def chartrepo(self,charts_namespace,charts_name,charts_version): file_url = self.url + "/chartrepo/{charts_namespace}/charts/{charts_name}-{charts_version}.tgz".format( charts_namespace=charts_namespace, charts_name=charts_name, charts_version=charts_version, ) # /root/test/{service}-{version}.tgz target_file = "{charts_name}-{charts_version}.tgz".format( charts_name=charts_name, charts_version=charts_version, ) try: wget.download(file_url, out=target_file) cmd = "helm template {target_file} | grep -v '^[[:space:]]*#' | grep image: | grep -o -E '/[a-z0-9]([-/a-z0-9]*[a-z0-9])?:[a-zA-Z0-9]([-._a-zA-Z0-9]*[a-zA-Z0-9])?' | sed 's@^/@@g' | grep -v '^op/lion:' | grep -v '^op/lion-init:' | grep -v '^op/db-init:'| sort | uniq".format( target_file=target_file, ) obj=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE) res = obj.stdout.read().decode('utf-8').split("\n")[0:-1] applist = [] for myapp in res: app_namespace = myapp.split("/")[0] app_service = myapp.split("/")[1].split(":")[0] app_version = myapp.split("/")[1].split(":")[1] app_info = { "app_service": app_service, "app_version": app_version, "app_namespace": app_namespace, } applist.append(app_info) charts_info = { charts_name: { "charts_version": charts_version, "charts_namespace": charts_namespace }, "service": applist } self.charts_list.append(charts_info) except Exception as e: with open('error_url.txt', mode='a', encoding='utf-8') as f: f.write(file_url) os.remove(target_file) def write_file(self,result): res=json.dumps(result) with open('chartinfo.json','w',encoding='utf-8') as f: f.write(res) if __name__ == "__main__": with open('image.txt', mode='w', encoding='utf-8') as f: f.write('') harbor_url = "https://harbor-test.eniot.io" res = HelmApi(harbor_url) res.charts_info()
六、实现上面的功能，不依赖helm命令的镜像过滤
镜像格式：image: "{{ .Values.global.harbor.host }}/enos/{{ .Chart.Name }}:tag_myapp_202009A"
#!/usr/bin/env python # -*- coding: UTF-8 -*- import wget import openpyxl import os import shutil import tarfile class HarborApi(): def __init__(self, url): self.url = url self.charts_list = [] def charts_info(self): wb = openpyxl.load_workbook('chart_info.xlsx') sh = wb['Sheet1'] for cases in list(sh.rows)[1:]: charts_namespace = cases[1].value charts_name = cases[2].value charts_version = cases[3].value chartinfo = self.chartrepo(charts_namespace, charts_name,charts_version) self.charts_files(chartinfo) def chartrepo(self,charts_namespace,charts_name,charts_version): chartinfo = {} file_url = self.url + "/chartrepo/{charts_namespace}/charts/{charts_name}-{charts_version}.tgz".format( charts_namespace=charts_namespace, charts_name=charts_name, charts_version=charts_version, ) # /root/test/{service}-{version}.tgz target_file = "{charts_name}-{charts_version}.tgz".format( charts_name=charts_name, charts_version=charts_version, ) try: wget.download(file_url, out=target_file) tar = tarfile.open(target_file) tar.extractall() tar.close() with tarfile.open(target_file, 'r') as t: for member_info in t.getmembers(): charts_files = member_info.name.split("/")[0] chartinfo['charts_files'] = charts_files chartinfo['charts_namespace'] = charts_namespace chartinfo['charts_name'] = charts_name chartinfo['charts_version'] = charts_version chartinfo['target_file'] = target_file return chartinfo except Exception as e: with open('error_url.txt', mode='a', encoding='utf-8') as f: f.write(file_url) f.write('\n') def charts_files(self,chartinfo): myset = {"deployment.yaml", "statefulset.yaml"} applist_address = chartinfo['charts_files'] + '/' + 'charts' applist = os.listdir(applist_address) for app in applist: app_address=applist_address+ '/' + app + '/' + 'templates' app_files = set(os.listdir(app_address)) yamlfiles = list(myset & app_files) if len(yamlfiles) > 0: yaml_address = app_address + '/' + yamlfiles[0] with open(yaml_address,mode='r',encoding='utf-8') as f: for line in f: if "image:" in line: app_namespace = line.split("/")[1] app_service = app app_version = line.split("/")[-1].split(":")[1].split("\"\n")[0] imageaddress = app_namespace + '/' +app_service+ ':' + app_version with open('image.txt', mode='a', encoding='utf-8') as f: f.write(imageaddress) f.write('\n') break os.remove(chartinfo['target_file']) shutil.rmtree(chartinfo['charts_files']) if __name__ == "__main__": with open('image.txt', mode='w', encoding='utf-8') as f: f.write('') harbor_url = "https://harbor-cn2.eniot.io" res = HarborApi(harbor_url) res.charts_info()
体现chart和images的关系
#!/usr/bin/env python # -*- coding: UTF-8 -*- import wget import openpyxl import os import shutil import tarfile import json import sys class HarborApi(): def __init__(self, url): self.url = url self.charts_list = [] def charts_info(self,savefiles): wb = openpyxl.load_workbook('chart_info.xlsx') sh = wb['Sheet1'] for cases in list(sh.rows)[1:]: charts_namespace = cases[1].value charts_name = cases[2].value charts_version = cases[3].value chartinfo = self.chartrepo(charts_namespace, charts_name,charts_version) self.charts_files(chartinfo,savefiles) def chartrepo(self,charts_namespace,charts_name,charts_version): chartinfo = {} file_url = self.url + "/chartrepo/{charts_namespace}/charts/{charts_name}-{charts_version}.tgz".format( charts_namespace=charts_namespace, charts_name=charts_name, charts_version=charts_version, ) # /root/test/{service}-{version}.tgz target_file = "{charts_name}-{charts_version}.tgz".format( charts_name=charts_name, charts_version=charts_version, ) try: wget.download(file_url, out=target_file) tar = tarfile.open(target_file) tar.extractall() tar.close() with tarfile.open(target_file, 'r') as t: for member_info in t.getmembers(): charts_files = member_info.name.split("/")[0] chartinfo['charts_files'] = charts_files chartinfo['charts_namespace'] = charts_namespace chartinfo['charts_name'] = charts_name chartinfo['charts_version'] = charts_version chartinfo['target_file'] = target_file return chartinfo except Exception as e: with open('error_url.txt', mode='a', encoding='utf-8') as f: f.write(file_url) f.write('\n') def charts_files(self,chartinfo,savefiles): myset = {"deployment.yaml", "statefulset.yaml"} applist_address = chartinfo['charts_files'] + '/' + 'charts' applist = os.listdir(applist_address) app_list_info = [] for app in applist: app_address=applist_address+ '/' + app + '/' + 'templates' app_files = set(os.listdir(app_address)) yamlfiles = list(myset & app_files) if len(yamlfiles) > 0: yaml_address = app_address + '/' + yamlfiles[0] with open(yaml_address,mode='r',encoding='utf-8') as f: for line in f: if "image:" in line: app_namespace = line.split("/")[1] app_service = app app_version = line.split("/")[-1].split(":")[1].split("\"\n")[0] app_info = { "app_service": app_service, "app_version": app_version, "app_namespace": app_namespace, } app_list_info.append(app_info) break charts_info = { chartinfo['charts_files']: { "charts_version": chartinfo['charts_version'], "charts_namespace": chartinfo['charts_namespace'], "service": app_list_info } } filename = savefiles + '/' +"%s.json" %chartinfo['charts_files'] result = json.dumps(charts_info) with open(filename, 'w', encoding='utf-8') as f: f.write(result) os.remove(chartinfo['target_file']) shutil.rmtree(chartinfo['charts_files']) if __name__ == "__main__": if len(sys.argv) == 2: savefiles = sys.argv[1] os.mkdir(savefiles) with open('image.txt', mode='w', encoding='utf-8') as f: f.write('') harbor_url = "https://harbor-test.eniot.io" res = HarborApi(harbor_url) res.charts_info(savefiles) else: print('please input save files')
