gitlab☞大版本迁移

阅读量: zyh 2018-09-06 20:14:54

Categories： > Tags：

前言

现在 gitlab 官方包升级很容易，直接 yum 之类的就可以，之所以有这篇文章，是因为我要从 v7 升级到 v11, 数据库也从 mysql 转变为了 pgsql;而且 v7 当时用的是 bitnami 安装的。所以无奈之下，就准备用 api 进行迁移，因而遗漏之处是肯定有的。

正文

Token变量定义[Token不同版本获取方式不一样，但是总归都是从web用户界面能直接拿到的], Token需要是最大权限用户，且拥有所有的项目权限
```
export OldToken=su2GwpNeVQ
export NewToken=yTVZTPeuss
export OldDomain=git.a.com
export NewDomain=git.b.com
```
gitlab 不同版本的 api 版本不一样，本文档是从 7 迁移到 11，所以 api 版本也是从 v3 到 v4
gitlab 的 api 默认一次最多获取100条信息，所以需要进行分页获取
per_page 每页信息条数 page 页数
所以信息太多的自行分页即可

迁移用户

下面代码，并不迁移用户的公钥信息

mkdir create_user && cd create_user;
for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/users?per_page=100&page=$i" | jq '.[] |.name, .username, .state, .email, .can_create_group, .can_create_project' | sed 'N;N;N;N;N;s#\n# #g;s#"##g';done >> users.list;


while read name username state email can_create_group can_create_project;do
    curl --header "PRIVATE-TOKEN:${NewToken}" --data "name=$name&username=$username&password=${username}@123456&state=$state&email=$email&can_create_group=$can_create_group&can_create_project=$can_create_project" "http://${NewDomain}/api/v4/users"
done < users.list
cd ..

创建项目组

mkdir create_group && cd create_group;
for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/groups?per_page=100&page=$i" | jq '.[] |.name, .path' | sed 'N;s#\n# #;s#"##g';done > group.list;
echo 'repo repo' >> group.list;

cat group.list | while read name path;do curl --request POST --header "PRIVATE-TOKEN:${NewToken}" --data "name=$name&path=$path" "http://${NewDomain}/api/v4/groups";done
cd ..

创建项目到对应组

mkdir create_project && cd create_project;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/projects?per_page=100&page=$i"  | jq '.[] |.name, .namespace.name, .description' | sed 'N;N;s#\n#\t#g;s# #-#g;s#"##g';done  >> project.list;

mv  project.list project.list.tmp;
cat project.list.tmp | sort | uniq > project.list;
rm -rf project.list.tmp;

# 这里 pre_page 不起作用
for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/groups?page=$i" | jq '.[] | .id, .name' | sed 'N;s#\n# #g;s#"##g';done >> group.id

cat project.list | while read projectname groupname des;do
    newgroupid=`egrep "\b${groupname}\b" ./group.id | awk '{print $1}'`
    curl --request POST --header "PRIVATE-TOKEN: ${NewToken}"  --data "name=${projectname}&namespace_id=${newgroupid}&description=$des" "http://${NewDomain}/api/v4/projects"
done
cd ..

添加用户到组

mkdir add_user_group && cd add_user_group;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/groups?page=$i" | jq '.[] | .id, .name' | sed 'N;s#\n# #g;s#"##g';done >> group.old.id;

cp ../create_project/group.id ./group.new.id;
#curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/groups&page=1" | jq '.[] | .id, .name' | sed 'N;s#\n# #g;s#"##g' >> group.new.id;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/users?per_page=100&page=$i"  | jq '.[] | .id,.username' | sed 'N;s#\n# #g;s#"##g';done >> new_user.id;

cat ./group.old.id | while read groupid groupname;do
    curl -s --header "PRIVATE-TOKEN: ${OldToken}" http://${OldDomain}/api/v3/groups/${groupid}/members | jq '.[] | .id, .username, .access_level' |sed 'N;N;s#\n# #g;s#"##g' | while read nameid username level;do
        newid=`egrep "\b${username}\b" ./new_user.id| awk '{print $1}'`
        newgid=`egrep "\b${groupname}\b" ./group.new.id|awk '{print $1}'`
	curl --request POST --header "PRIVATE-TOKEN: ${NewToken}" --data "user_id=${newid}&access_level=${level}" http://${NewDomain}/api/v4/groups/${newgid}/members
    done
done
cd ..

添加用户到项目

mkdir add_user_project && cd add_user_project;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/projects?per_page=100&page=$i"  | jq '.[] |.id, .name, .namespace.name' | sed 'N;N;s#\n# #g;s#"##g';done  >> project.old.id;

mv  project.old.id project.old.id.tmp;
cat project.old.id.tmp | sort | uniq > project.old.id;
rm -rf project.old.id.tmp;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/projects?per_page=100&page=$i"  | jq '.[] |.id, .name, .namespace.name' | sed 'N;N;s#\n# #g;s#"##g';done  >> project.new.id;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/users?per_page=100&page=$i"  | jq '.[] | .id,.username' | sed 'N;s#\n# #g;s#"##g';done >> new_user.id;

cat ./project.old.id | while read projectid projectname groupname;do
    curl -s --header "PRIVATE-TOKEN: ${OldToken}" http://${OldDomain}/api/v3/projects/${projectid}/members | jq '.[] | .id, .username, .access_level' |sed 'N;N;s#\n# #g;s#"##g' | while read nameid username level;do
        newid=`egrep "\b${username}\b" ./new_user.id| awk '{print $1}'`
        newpid=`egrep "\b${projectname} ${groupname}\b" ./project.new.id|awk '{print $1}'`
        #echo "$projectid $projectname $groupname $username > $newpid $projectname $groupname"
	curl --request POST --header "PRIVATE-TOKEN: ${NewToken}" --data "user_id=${newid}&access_level=${level}" http://${NewDomain}/api/v4/projects/${newpid}/members
    done
done
cd ..

代码迁移

mkdir sync_code && cd sync_code;
#for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/projects?per_page=100&page=$i"  | jq '.[] |.name, .namespace.name, .description' | sed 'N;N;s#\n#\t#g;s# #-#g;s#"##g';done  >> project.list;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/projects?per_page=100&page=$i" | jq '.[] |.path_with_namespace' | sed 's#"##g';done >> project.path;

# 将本代码框的代码保存到 start.sh 并执行(确保sync_code_base变量路径准确)
num=1
sync_code_base=/export/sync/sync_code
nohup while read line;do 
    echo "$num: -------------------------$line-------------------------" >> ${sync_code_base}/sync.log
    [[ -d ${line%%/*} ]] || mkdir ${line%%/*}
    cd ${line%%/*}
    git clone --mirror git@${OldDomain}:${line}.git >> ${sync_code_base}/sync.log 2>&1 || echo "clone:$line" >> ${sync_code_base}/clone_error.log
    cd ${line##*/}.git
    git remote set-url origin git@${NewDomain}:${line}.git
    git push -f origin >> ${sync_code_base}/sync.log 2>&1 || echo "sync:$line" >> ${sync_code_base}/sync_error.log
    cd /export/sync/sync_code
    let num++
done < ${sync_code_base}/project.path &

cd ..

初始化新用户密码

mkdir init_password; cd init_password;
for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/users?per_page=100&page=$i" |  jq '.[] |.id, .name, .username' |  sed 'N;N;s#\n# #g;s#"##g';done >> user.id;

cat user.id | while read id name nameuser;do  curl --request PUT --header "PRIVATE-TOKEN: ${NewToken}" --data "password=${nameuser}@123" http://${NewDomain}/api/v4/users/${id};done
cd ..

同步用户公钥信息

mkdir sync_user_pub && cd sync_user_pub;
for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${OldToken}" "http://${OldDomain}/api/v3/users?per_page=100&page=$i"  |  jq '.[] |.id, .name, .username' |  sed 'N;N;s#\n# #g;s#"##g';done >> old_users.id;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/users?per_page=100&page=$i"  |  jq '.[] |.id, .name, .username' |  sed 'N;N;s#\n# #g;s#"##g';done >> new_users.id;

cat old_users.id | while read old_current_id name username;do
    new_current_id=`grep $username new_users.id | awk '{print $1}'`
    curl -s --header "PRIVATE-TOKEN: ${OldToken}" http://${OldDomain}/api/v3/users/${old_current_id}/keys | jq '.[]|.title,.key' | sed 'N;s/\n/\t/;s/"//g' | while read  current_title current_key;do
        curl --request POST --header "PRIVATE-TOKEN:${NewToken}" --data-urlencode "title=${current_title}" --data-urlencode "key=${current_key}" "http://${NewDomain}/api/v4/users/${new_current_id}/keys"
    done
done

如果出现push故障，则删除新服务器的项目, 并重新push

sync_code_base=/export/sync/sync_code
cd ${sync_code_base}
# 准备好 delete.path 文件，内部一行一个 project.path_with_namespace

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/projects?per_page=100&page=$i" | jq '.[]|.id, .path_with_namespace'| sed 'N;s#\n#\t#g;s# #-#g;s#"##g';done > project-path-id.all

while read line;do grep $line project-path-id.all ;done < delete.path > delete.path.id

while read id path;do curl -s --request DELETE --header "PRIVATE-TOKEN:${NewToken}" http://${NewDomain}/api/v4/projects/${id};sleep 5;cd ${path}.git;git push -f origin ;cd ${sync_code_base};done < delete.path.id

删除新服务器的项目组

mkdir delete_group && cd delete_group;

for i in `seq 1 15`;do curl -s --header "PRIVATE-TOKEN: ${NewToken}" "http://${NewDomain}/api/v4/groups?per_page=100&page=$i" | jq '.[] | .id';done >> group.id;

cat group.id | while read id;do curl -s --request DELETE --header "PRIVATE-TOKEN: ${NewToken}" http://${NewDomain}/api/v4/groups/${id};done;

gitlab☞批量修改用户密码

nginx☞泛域名_变量截取