git设置
# git config --global http.postBuffer 1048576000
manifest.xml
原始的manifests.xml文件在源头代码repo仓库中
# ls -l .repo/
total 40
drwxrwxr-x 5 1001 1001 4096 Dec 9 14:37 manifests
drwxrwxr-x 10 1001 1001 4096 Dec 2 14:10 manifests.git
lrwxrwxrwx 1 1001 1001 56 Jan 28 11:25 manifest.xml -> manifests/openlinux/bds/r_amlogic_t7_20221221_64_bit.xml
-rw-rw-r-- 1 1001 1001 18173 Jan 28 14:05 project.list
drwxrwxr-x 17 1001 1001 4096 Jan 28 13:02 project-objects
drwxrwxr-x 31 1001 1001 4096 Jan 28 13:02 projects
drwxrwxr-x 7 1001 1001 4096 Nov 19 14:16 repo
批量创建服务端空仓库
创建脚本
import json
import re
import requests
from collections import Counter
filename = 'manifest.xml'
pattern = '<project name=[\\w\\-./]*'
root_group = 'r-amlogic'
git_url = 'https://project.example.com:1443'
headers = {"PRIVATE-TOKEN": "xxxxxxxxxxxx", "Content-Type": "application/json"}
groups_api = git_url + '/api/v4/groups'
projects_api = git_url + '/api/v4/projects/'
# 获取服务端gitlab组信息
def gitlab_group_info():
exist_group_lists = {}
projects_pages = int(requests.head(groups_api + '?page=100', headers=headers).headers['X-Total-Pages'])
for i in range(projects_pages):
page = i + 1
response = requests.request("get", groups_api + '?page=100' + '&page=' + str(page), headers=headers)
with response:
for group in response.json():
exist_group_lists['/' + group['full_path']] = group['id']
return exist_group_lists # {'/gitlab-instance-0b80b373': 2, '/wgs': 17}
# 创建gitlab 组 返回group_id
def create_gitlab_group(name, path, parent_id=None):
group_lists = gitlab_group_info()
if path not in group_lists.keys():
response = requests.request("post", groups_api,
data=json.dumps({"path": path, "name": name, "parent_id": parent_id}),
headers=headers)
with response:
group_info = response.json()
return group_info
# 获取服务端projects信息
def group_projects_info():
project_lists = {}
root_group_id = gitlab_group_info()['/' + root_group]
groups_projects_api = git_url + '/api/v4/groups/' + str(root_group_id) + '/projects'
projects_pages = int(
requests.head(groups_projects_api + '?page=100' + '&include_subgroups=true', headers=headers).headers[
'X-Total-Pages'])
for i in range(projects_pages):
page = i + 1
response = requests.request("get",
groups_projects_api + '?page=100' + '&include_subgroups=true' + '&page=' + str(
page), headers=headers)
with response:
for group in response.json():
project_lists['/' + group['path_with_namespace']] = group['id']
return project_lists # {'/r-amlogic/vendor/widevine': 3288, '/r-amlogic/vendor/external/freetype': 3287}
# 创建project
def create_gitlab_project(name, namespace_id):
response = requests.request("post", projects_api, headers=headers,
data=json.dumps({"name": name, "namespace_id": namespace_id}))
with response:
projects_info = response.json()
return projects_info
# 在manifest.xml文件中获取gitlab项目的url
def project_and_group_path():
total_project_lists = []
some_name = dict()
with open(filename, 'r') as f:
regex = re.compile(pattern)
result = regex.findall(f.read().replace('"', ""))
for project_path in result:
total_project_lists = total_project_lists + [
'/' + root_group + '/' + project_path.replace('<project name=', '')]
_project_lists = dict(Counter(total_project_lists))
duplicate_project = [key for key, value in _project_lists.items() if value > 1]
project_lists = list(set(total_project_lists))
print('本地文件总project数量为{}个,重复项目数量{}个,需要创建的项目为{}个'
.format(len(total_project_lists), len(duplicate_project), len(total_project_lists) - len(duplicate_project)))
if len(duplicate_project) > 0:
print('重复项目是{}'.format(duplicate_project))
for _project_list in project_lists:
if _project_list.rsplit(sep='/', maxsplit=1)[0] in project_lists:
if _project_list.rsplit(sep='/', maxsplit=1)[0] not in some_name.keys():
some_name[_project_list.rsplit(sep='/', maxsplit=1)[0]] = list()
some_name[_project_list.rsplit(sep='/', maxsplit=1)[0]] = some_name[
_project_list.rsplit(sep='/', maxsplit=1)[
0]] + [_project_list]
if some_name:
print('存在group和project名称相同的路径，需要修改project名称')
print(some_name)
exit()
return project_lists # ['/r-amlogic/platform/external/jsr305', '/r-amlogic/platform/hardware/amlogic/tuner']
def create_project_and_group():
exist_group_lists = gitlab_group_info()
project_lists = project_and_group_path()
for _group_list in project_lists:
group_path = []
group_list = _group_list.split('/') # ['r-amlogic', 'DTVKit', 'android-inputsource']
if '' in group_list:
group_list.remove('')
if len(group_list) > 2:
for group_i in range(len(group_list) - 1):
if group_i == 0:
group_path.append(['/' + group_list[group_i]]) # [['/r-amlogic']]
else:
group_path.append(
[group_path[group_i - 1][0] + '/' + group_list[
group_i]]) # [['/r-amlogic'], ['/r-amlogic/DTVKit']]
if group_path[group_i][0] not in exist_group_lists.keys():
if group_list[group_i] == root_group:
parent_id = None
else:
parent_id = exist_group_lists[group_path[group_i - 1][0]]
create_gitlab_group(group_list[group_i], group_list[group_i], parent_id=parent_id)
exist_group_lists = gitlab_group_info()
create_gitlab_project(group_list[-1], exist_group_lists[group_path[-1][0]])
if len(group_list) == 2:
if '/' + root_group not in exist_group_lists.keys():
create_gitlab_group(root_group, root_group, parent_id=None)
parent_id = exist_group_lists['/' + root_group]
create_gitlab_project(group_list[-1], parent_id)
if __name__ == "__main__":
create_project_and_group()
修改manifest.xml
创建脚本会检测group名称和project名称同名的行， 按需修改相应的行
push本地项目到gitlab
仓库说明
仓库克隆的方式存在 git clone --depth, 此仓库无法push到新服务端，需要单独处理
push正常的仓库
使用上一步修改过的manifest.xml文件
#!/bin/bash
IFS=$'\n'
manifest_path=/data/manifest.xml
project_local_root_path=/data/r_amlogic_a311d2_t982
user=root
token=xxxxxxxxxx
gitlab_url=https://${user}:${token}@project.example.com:1443/r-amlogic
git config --global user.name "Administrator"
git config --global user.email "1304995320@qq.com"
echo '' > ${project_local_root_path}/fail.txt
for line in `grep "<project" ${manifest_path}`;do
url_name=`echo ${line} | egrep -o "name=[-\"._a-zA-Z0-9/]*" | awk -F'=' '{print $2}' | sed 's@"@@g'`
local_path=`echo ${line} | egrep -o "path=[-\"._a-zA-Z0-9/]*" | awk -F'=' '{print $2}' | sed 's@"@@g'`
local_remote=`echo ${line} | egrep -o "remote=[-\"._a-zA-Z0-9/]*" | awk -F'=' '{print $2}' | sed 's@"@@g'`
[ ! ${local_path} ] && local_path=${url_name}
[ ! ${local_remote} ] && local_remote=aosp
cd ${project_local_root_path}/${local_path}
git remote set-url ${local_remote} ${gitlab_url}/${url_name}.git
git branch r-amlogic
git checkout r-amlogic
git push -u ${local_remote} --all
if [ $? -ne 0 ];then
echo " ${url_name} ${local_path} " >> ${project_local_root_path}/fail.txt
fi
git push -u ${local_remote} --tags
if [ $? -ne 0 ];then
echo " ${url_name} ${local_path} " >> ${project_local_root_path}/fail.txt
fi
done
push depth方式clone的仓库
根据上一步生成的fail.txt，查看失败的仓库的路径
初始化仓库
rm -rf .git
git init
git switch -c r-amlogic
git remote add ${local_remote} ${gitlab_url}/${url_name}.git
git add .
git commit -m "Initial commit"
git push -u ${local_remote} r-amlogic
更新该文件的revision="5ac02a0ba9f39591758bab4516dd0725747af0c3"
署清单仓库manifests.git
## 克隆代码
git clone .../manifests.git
## 提交清单文件manifest.xml
git add manifest.xml
git commit -m "add manifest.xml"
git push
初始化仓库
部署repo
# curl https://mirrors.tuna.tsinghua.edu.cn/git/git-repo -o repo
# chmod +x repo
初始化仓库
# export REPO_URL=http://mirrors.tuna.tsinghua.edu.cn/git/git-repo/
# ./repo init -u ssh://git@project.example.com:35022/r-amlogic/manifests.git -b main -m manifest.xml
Downloading Repo source from http://mirrors.tuna.tsinghua.edu.cn/git/git-repo/
remote: Enumerating objects: 4439, done.
remote: Counting objects: 100% (4439/4439), done.
remote: Compressing objects: 100% (2085/2085), done.
remote: Total 7925 (delta 3963), reused 2354 (delta 2354), pack-reused 3486
Receiving objects: 100% (7925/7925), 3.57 MiB | 2.49 MiB/s, done.
Resolving deltas: 100% (5123/5123), done.
Your identity is: Administrator <wangguishe@hard-chain.cn>
If you want to change this, please re-run 'repo init' with --config-name
Testing colorized output (for 'repo diff', 'repo status'):
black red green yellow blue magenta cyan white
bold dim ul reverse
Enable color display in this user account (y/N)? y
repo has been initialized in /data/apps
查看生成目录
# ls -l /data/apps/.repo/
total 28
drwxr-xr-x 2 root root 4096 Mar 14 15:26 manifests
drwxr-xr-x 8 root root 4096 Mar 14 15:26 manifests.git
-rw-r--r-- 1 root root 501 Mar 14 15:26 manifest.xml
drwxr-xr-x 11 root root 4096 Mar 14 15:26 repo
-rw-r--r-- 1 root root 11316 Mar 14 15:28 TRACE_FILE
同步project
# ./repo sync
Fetching: 100% (781/781), done in 9m46.941s
....
Checking out: 100% (781/781), done in 4m22.951s
repo sync has finished successfully.
报错处理
问题一
报错信息
fatal: the remote end hung up unexpectedly
解决办法
修改提交缓存大小为500M，或者更大的数字
git config --global http.postBuffer 1048576000
问题二
报错信息
! [remote rejected] HEAD -> android-11.0.0_r40 (shallow update not allowed)
解决办法
克隆代码方式为 git clone --depth
需要获取历史记录
git fetch --unshallow origin
不需要历史记录
rm -rf .git
git init
问题三
报错信息
Delta compression using up to 16 threads
Compressing objects: 100% (9593/9593), done.
fatal: protocol error: bad line length 8192iB | 166.89 MiB/s
error: failed to push some refs to
解决办法
# git config http.postBuffer 52428800
问题四
报错信息
remote: GitLab: LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".
To https://xxxxx:1443/r-amlogic/platform/external/OpenCL-CTS.git
! [remote rejected] m/r-amlogic -> m/r-amlogic (deny updating a hidden ref)
! [remote rejected] android-11.0.0_r27 -> android-11.0.0_r27 (pre-receive hook declined)
! [remote rejected] android-11.0.0_r29 -> android-11.0.0_r29 (pre-receive hook declined)
解决办法
