GitLab Python Clone Everything with Structure

Question

I want to clone all of the projects in my GitLab environment. There are a number of groups and subgroups with their own projects. I'd like to clone the projects while maintaining the structure of the groups locally. For example:

Group1
  Subgroup1
     Project1
Group2
  Project1
Group3
  Subgroup1
     Project1
  Subgroup2
     Project1
     Project2

So if I run cd Group3/Subgroup2/Project2 locally I would be in that respective projects code.

My code does some of the work but it breaks on subgroups:

import os
from git import Repo

GITLAB_TOKEN = os.getenv('GITLAB_TOKEN')

gl = gitlab.Gitlab(url='https://gitlab.example.priv', private_token=GITLAB_TOKEN)

all_groups = gl.groups.list(all=True,top_level_only=True)
count=0
for group in all_groups:
    os.chdir('/Users/ken/repos')
    print("---------------- {} ----------------".format(group.full_path))
    if not os.path.exists(group.full_path):
        print("\tCreating directory for group " + group.full_path)
        os.makedirs(group.full_path)
    os.chdir(group.full_path)
    print("Cloning projects for group " + group.full_path)
    for project in group.projects.list(all=True):
        if not os.path.exists(project.path):
            Repo.clone_from(project.ssh_url_to_repo, project.path)
    print("Creating directories for subgroups")
    subgroups = group.subgroups.list()
    for subgroup in subgroups:
        if not os.path.exists(subgroup.full_path):
            os.makedirs(subgroup.full_path)
        os.chdir(subgroup.full_path)
        print("Cloning projects for subgroup " + subgroup.full_path)
        group = gl.groups.get(subgroup.id, lazy=True)
        for project in group.projects.list(all=True):
            Repo.clone_from(project.ssh_url_to_repo, project.path)
        os.chdir('..')

How can I iterate over groups and subgroups to clone all of the projects while maintaining the folder structure locally?

I ended up calling the projects directly and using their respective namespaces to create the pathToFolder. So rather than cd'ing in and out just create the absolute paths. — user3063045, Jan 08 '22 at 12:37

score 1 · Answer 1 · answered Jan 08 '22 at 12:39

I ended up calling the projects directly and using their respective namespaces to create the pathToFolder. So rather than cd'ing in and out just create the absolute paths.

import os
import gitlab
from git import Repo

GITLAB_TOKEN = os.getenv('GITLAB_TOKEN')

gl = gitlab.Gitlab(url='https://gitlab.example.priv', private_token=GITLAB_TOKEN)

gitBasePathRelative = "repos/"
gitBasePathRelativeAbsolut = os.path.expanduser("~/" + gitBasePathRelative)
os.makedirs(gitBasePathRelativeAbsolut,exist_ok=True)

for p in gl.projects.list(all=True):
    print("Cloning project " + p.name)
    pathToFolder = p.namespace['name'] + "/" + p.name
    if not os.path.exists(pathToFolder):
        print("\tCreating directory for project " + pathToFolder)
        os.makedirs(pathToFolder)
        Repo.clone_from(p.ssh_url_to_repo, pathToFolder)

GitLab Python Clone Everything with Structure

1 Answers1