From ae6624530581aa34aba471e226525a9714bcd484 Mon Sep 17 00:00:00 2001
From: Maximilian Kratz <account@maxkratz.com>
Date: Sun, 15 Jan 2023 12:49:58 +0100
Subject: [PATCH] Major extensions of the mirror script + adds a script for
 orga deletion

Smaller doc corrections
More code-style corrections
Some progress to fetch organization repos
First basic orga mirroring is working now
Parameter parsing + robustness if orga already exists
Clean up
New argument parsing + configurable visibility for orga creation
Snapshot: Possibility to generate private mirrors from orgas
Adds functionality to allow the mirroring of one private/public repo
Doc + pretty print
Fixes a return code check + code style
Adds a script to completely delete a Gitea organization
Fixes a var underflow
Mirror a complete GitHub user
Also include private repos of the user
Fixes a loop bug in repos_to_migration
Typo
---
 delete_gitea_org    | 109 +++++++++++++++++++
 github2gitea-mirror | 251 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 331 insertions(+), 29 deletions(-)
 create mode 100755 delete_gitea_org

diff --git a/delete_gitea_org b/delete_gitea_org
new file mode 100755
index 0000000..747705d
--- /dev/null
+++ b/delete_gitea_org
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+#
+# Script to completely delete an organization with all its repos on a Gitea instance.
+#
+# Heavily inspired by:
+#   https://github.com/juergenhoetzel/github2gitea-mirror
+# 
+
+# ENVs:
+#   ACCESS_TOKEN = Gitea token
+#   GITERA_URL   = Gitea URL
+
+# Displays the given input including "=> " on the console.
+log () {
+	echo "=> $1"
+}
+
+CURL="curl -f -S -s"
+
+# Check for correctly set ENVs
+# ACCESS_TOKEN and GITEA_URL are always necessary
+if [[ -z "${ACCESS_TOKEN}" || -z "${GITEA_URL}" ]]; then
+    echo -e "Please set the Gitea access token and URL in environment:\nexport ACCESS_TOKEN=abc\nexport GITEA_URL=http://gitea:3000\n" >&2
+    echo -e "Don't use a trailing slash in URL!"
+    exit 1
+fi
+
+# Parse input arguments
+if [[ -z "$1" ]]; then
+	log "No parameter(s) given. Exit."
+    exit 1
+fi
+while [[ "$#" -gt 0 ]]; do
+	case $1 in
+		-o|--org) gitea_organization="$2"; shift ;;
+		*) log "Unknown parameter passed: $1"; exit 1 ;;
+	esac
+	shift
+done
+
+# Prints a message on how to use the script with exit 1
+fail_print_usage () {
+    echo -e "Usage: $0"
+    echo -e "   -o, --org \$organization             GitHub organization to mirror and/or the target organization in Gitea."
+    echo "" >&2
+    exit 1;
+}
+
+if [[ -z "${gitea_organization}" ]]; then
+    echo -e "Organization not set."
+    fail_print_usage
+fi
+
+# TODO:
+#set -euo pipefail
+set -eu pipefail
+
+header_options=(-H  "Authorization: Bearer ${ACCESS_TOKEN}" -H "accept: application/json" -H "Content-Type: application/json")
+jsonoutput=$(mktemp -d -t github-repos-XXXXXXXX)
+
+trap "rm -rf ${jsonoutput}" EXIT
+
+# Fetches all public/private repos of the given Gitea organization to '1.json'
+fetch_orga_repos() {
+    log "Fetch organization repos."
+    if ! $CURL -X GET $GITEA_URL/api/v1/orgs/${gitea_organization}/repos "${header_options[@]}" >${jsonoutput}/1.json 2>${jsonoutput}/stderr.txt; then
+        local code=$(<${jsonoutput}/result.txt)
+        if (( code != 404 ));then # 404 == orga not found
+            cat ${jsonoutput}/stderr.txt >&2
+        fi
+    fi
+}
+
+delete_orga_repos() {
+    log "Delete orga repos."
+    for f in ${jsonoutput}/1.json; do
+        n=$(jq '. | length'<$f)
+        if [[ "${n}" -gt "0" ]]; then
+            (( n-- )) # last element
+        else
+            break;
+        fi
+        echo "Deleting $n repos."
+        for i in $(seq 0 $n); do
+            del_user=$(jq -r ".[$i] | .owner.username" <$f)
+            del_repo=$(jq -r ".[$i] | .name " <$f)
+            echo "Deleting repo: $del_user/$del_repo"
+            $CURL -X DELETE $GITEA_URL/api/v1/repos/${del_user}/${del_repo} "${header_options[@]}" > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt
+        done
+    done
+}
+
+delete_orga() {
+    log "Delete orga."
+    if ! $CURL -X DELETE $GITEA_URL/api/v1/orgs/${gitea_organization} "${header_options[@]}" > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
+        local code=$(<${jsonoutput}/result.txt)
+        if (( code != 404 ));then # 404 == orga not found
+            cat ${jsonoutput}/stderr.txt >&2
+        fi
+    fi
+}
+
+# Actual run the script
+fetch_orga_repos
+delete_orga_repos
+delete_orga
+
+log "Finished."
diff --git a/github2gitea-mirror b/github2gitea-mirror
index b262a9b..58e2180 100755
--- a/github2gitea-mirror
+++ b/github2gitea-mirror
@@ -1,59 +1,252 @@
 #!/bin/bash
 
-set -euo pipefail
+#
+# Script to mirror GitHub repos to a Gitea instance.
+#
+# Modes:
+#   - Mirror a public/private repo
+#   - Mirror all public/private repos of a user
+#   - Mirror all starred repos by a user
+#   - Mirror all public/private repos of an organization
+#
+# Heavily inspired by:
+#   https://github.com/juergenhoetzel/github2gitea-mirror
+# 
+
+# ENVs:
+#   ACCESS_TOKEN = Gitea token
+#   GITERA_URL   = Gitea URL
+#   GITHUB_TOKEN = GitHub personal access token
+
+# Displays the given input including "=> " on the console.
+log () {
+	echo "=> $1"
+}
 
-# Mirror starred Github repos to a Gitea Organization
 CURL="curl -f -S -s"
 
-if [[ -z "${ACCESS_TOKEN}" || -z "${GITEA_URL}" ]];then
-    echo -e "Please set gitea access token and url in environment:\nexport ACCESS_TOKEN=abc\nexport GITEA_URL=http://gitea:3000\n" >&2
-    echo -e "Don't use trailing slash in URL!"
+# Check for correctly set ENVs
+# ACCESS_TOKEN and GITEA_URL are always necessary
+if [[ -z "${ACCESS_TOKEN}" || -z "${GITEA_URL}" ]]; then
+    echo -e "Please set the Gitea access token and URL in environment:\nexport ACCESS_TOKEN=abc\nexport GITEA_URL=http://gitea:3000\n" >&2
+    echo -e "Don't use a trailing slash in URL!"
     exit 1
 fi
 
-github_user="${1:-}"
-gitea_organization="${2:-}"
-
-if [[ -z "${github_user}" || -z "${gitea_organization}" ]]; then
-    echo "Usage: $0 github_user gitea_organization" >&2
+# Parse input arguments
+if [[ -z "$1" ]]; then
+	log "No parameter(s) given. Exit."
     exit 1
 fi
+while [[ "$#" -gt 0 ]]; do
+	case $1 in
+		-m|--mode) mode="$2"; shift ;;
+		-o|--org) gitea_organization="$2"; shift ;;
+        -u|--user) github_user="$2"; shift ;;
+        -v|--visibility) visibility="$2"; shift ;;
+        -r|--repo) repo="$2"; shift ;;
+		*) log "Unknown parameter passed: $1"; exit 1 ;;
+	esac
+	shift
+done
+
+# Prints a message on how to use the script with exit 1
+fail_print_usage () {
+    echo -e "Usage: $0"
+    echo -e "   -m, --mode {org,star,repo,user}     Mode to use; either mirror an organization or mirror all starred repositories."
+    echo -e "   -o, --org \$organization             GitHub organization to mirror and/or the target organization in Gitea."
+    echo -e "   -u, --user \$github_user             GitHub user to gather the starred repositories from."
+    echo -e "   -v, --visibility {public,private}   Visibility for the created Gitea organization."
+    echo -e "   -r, --repo \$repo_url                GitHub URL of a single repo to create a mirror for."
+    echo "" >&2
+    exit 1;
+}
+
+# Check if mode is set
+if [[ -z "${mode}" ]]; then
+    fail_print_usage
+fi
+
+# Check required parameters per mode
+if [ "${mode}" == "org" ]; then
+    if [[ -z "${gitea_organization}" ]]; then
+        echo -e "Organization not set."
+        fail_print_usage
+    fi
+
+    if [[ -z "${visibility}" ]]; then
+        echo -e "Visibility not set."
+        fail_print_usage
+    fi
+elif [ "${mode}" == "star" ]; then
+    if [[ -z "${gitea_organization}" || -z "${github_user}" ]]; then
+        echo -e "Organization or GitHub user not set."
+        fail_print_usage
+    fi
+elif [ "${mode}" == "repo" ]; then
+    if [[ -z "${repo}" || -z "${github_user}" ]]; then
+        echo -e "Repo URL or GitHub user not set."
+        fail_print_usage
+    fi
+elif [ "${mode}" == "user" ]; then
+    if [[ -z "${github_user}" ]]; then
+        echo -e "GitHub user not set."
+        fail_print_usage
+    fi  
+else
+    echo -e "Mode not found."
+    fail_print_usage
+fi
+
+# TODO:
+#set -euo pipefail
+set -eu pipefail
 
 header_options=(-H  "Authorization: Bearer ${ACCESS_TOKEN}" -H "accept: application/json" -H "Content-Type: application/json")
 jsonoutput=$(mktemp -d -t github-repos-XXXXXXXX)
-trap "rm -rvf ${jsonoutput}" EXIT
 
-uid=$($CURL "${header_options[@]}" $GITEA_URL/api/v1/orgs/${gitea_organization} |jq .id)
+trap "rm -rf ${jsonoutput}" EXIT
 
-fetch_starred_repos(){
+# Sets the uid to the specified Gitea organization
+set_uid() {
+    uid=$($CURL "${header_options[@]}" $GITEA_URL/api/v1/orgs/${gitea_organization} | jq .id)
+}
+
+# Sets the uid to the specified Gitea user
+set_uid_user() {
+    uid=$($CURL "${header_options[@]}" $GITEA_URL/api/v1/users/${github_user} | jq .id)
+}
+
+# Fetches all starred repos of the given user to JSON files
+fetch_starred_repos() {
+    log "Fetch starred repos."
     i=1
-    # Github API just returns empty arrays instead of 404
+    # GitHub API just returns empty arrays instead of 404
     while $CURL "https://api.github.com/users/${github_user}/starred?page=${i}&per_page=100" >${jsonoutput}/${i}.json \
-	    && (( $(jq <${jsonoutput}/${i}.json '.|length') > 0 )) ; do
-	(( i++ ))
+	    && (( $(jq <${jsonoutput}/${i}.json '. | length') > 0 )) ; do
+	    (( i++ ))
     done
 }
 
+# Fetches all public/private repos of the given GitHub organization to JSON files
+fetch_orga_repos() {
+    log "Fetch organization repos."
+    i=1
+    # GitHub API just returns empty arrays instead of 404
+    while $CURL "https://api.github.com/orgs/${gitea_organization}/repos?page=${i}&per_page=100" -u "username:${GITHUB_TOKEN}" >${jsonoutput}/${i}.json \
+	    && (( $(jq <${jsonoutput}/${i}.json '. | length') > 0 )) ; do
+	    (( i++ ))
+    done
+}
+
+# Fetches all public/private repos of the given GitHub user to JSON files
+fetch_user_repos() {
+    log "Fetch user repos."
+    i=1
+    # GitHub API just returns empty arrays instead of 404
+    while $CURL "https://api.github.com/user/repos?affiliation=owner&page=${i}&per_page=100" -u "${github_user}:${GITHUB_TOKEN}" >${jsonoutput}/${i}.json \
+	    && (( $(jq <${jsonoutput}/${i}.json '. | length') > 0 )) ; do
+	    (( i++ ))
+    done
+}
+
+# Fetches one public/private GitHub repo to a JSON file
+fetch_one_repo() {
+    log "Fetch one repo."
+    # Remove URL prefix
+    repo=$(echo $repo | sed "s/https:\/\/github.com\///g" | sed "s/.git//g")
+    $CURL "https://api.github.com/repos/$repo" -u "username:${GITHUB_TOKEN}" >${jsonoutput}/1.json
+}
+
+# Creates a specific migration repo on Gitea
 create_migration_repo() {
-    if ! $CURL -w  "%{http_code}\n"  "${header_options[@]}" -d @- -X POST $GITEA_URL/api/v1/repos/migrate > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt;then
-	local code=$(<${jsonoutput}/result.txt)
-	if (( code != 409 ));then # 409 == repo already exits
-	    cat ${jsonoutput}/stderr.txt >&2
-	fi
+    log "Create migration repo."
+    if ! $CURL -w  "%{http_code}\n"  "${header_options[@]}" -d @- -X POST $GITEA_URL/api/v1/repos/migrate > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
+        local code=$(<${jsonoutput}/result.txt)
+        if (( code != 409 ));then # 409 == repo already exits
+            cat ${jsonoutput}/stderr.txt >&2
+        fi
     fi
 }
 
+# Creates a specific public/private organization on Gitea
+create_migration_orga() {
+    visibility="${1:-}"
+    log "Create migration orga with name: ${gitea_organization}"
+    if ! $CURL -X POST $GITEA_URL/api/v1/orgs "${header_options[@]}" --data '{"username": "'"${gitea_organization}"'", "visibility": "'"${visibility}"'"}' > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
+        local code=$(<${jsonoutput}/result.txt)
+        if (( code != 422 ));then # 422 == orga already exits
+            cat ${jsonoutput}/stderr.txt >&2
+        fi
+    fi
+}
+
+# Creates a migration repo on Gitea for each GitHub repo in the JSON files
 repos_to_migration() {
+    log "Repos to migration started."
     for f in ${jsonoutput}/*.json; do
-	n=$(jq '.|length'<$f)
-	(( n-- ))		# last element
-	for i in $(seq 0 $n); do
-	    jq ".[$i]|.uid=${uid}|.mirror=true|.clone_addr=.clone_url|.description=.description[0:255]|.repo_name=.name|{uid,repo_name,clone_addr,description,mirror}" <$f \
-		| create_migration_repo
-	done
+        n=$(jq '. | length'<$f)
+        if [[ "${n}" -gt "0" ]]; then
+            (( n-- )) # last element
+        else
+            continue;
+        fi
+        for i in $(seq 0 $n); do
+            mig_data=$(jq ".[$i] | .uid=${uid} | \
+                if(.visibility==\"private\") then .private=true else .private=false end |\
+                if(.visibility==\"private\") then .auth_username=\"${github_user}\" else . end | \
+                if(.visibility==\"private\") then .auth_password=\"${GITHUB_TOKEN}\" else . end | \
+                .mirror=true | \
+                .clone_addr=.clone_url | \
+                .description=.description[0:255] | \
+                .repo_name=.name | \
+                {uid,repo_name,clone_addr,description,mirror,private,auth_username,auth_password}" <$f)
+            echo "Migrating repo" $(jq ".[$i] | .uid=${uid} | .name" <$f)
+            echo $mig_data | create_migration_repo
+        done
     done
 }
 
-fetch_starred_repos
-repos_to_migration
+# Creates one migration repo on Gitea for the one GitHub repo in '1.json'
+one_repo_to_migration() {
+    log "One repo to migration started."
+    # There should only be one JSON file
+    for f in ${jsonoutput}/*.json; do
+        mig_data=$(jq ".repo_owner=\"${github_user}\" | \
+            if(.visibility==\"private\") then .private=true else .private=false end |\
+            if(.visibility==\"private\") then .auth_username=\"${github_user}\" else . end | \
+            if(.visibility==\"private\") then .auth_password=\"${GITHUB_TOKEN}\" else . end | \
+            .mirror=true | \
+            .clone_addr=.clone_url | \
+            .description=.description[0:255] | \
+            .repo_name=.name | \
+            {repo_owner,repo_name,clone_addr,description,mirror,private,auth_username,auth_password}" <$f)
+        echo "Migrating repo" $(jq ".name" <$f)
+        echo $mig_data | create_migration_repo
+    done
+}
 
+# Actual run the script
+if [ "${mode}" == "org" ]; then
+    log "Mode = organization"
+    fetch_orga_repos
+    create_migration_orga ${visibility}
+    set_uid
+    repos_to_migration
+elif [ "${mode}" == "repo" ]; then
+    log "Mode = single repo"
+    fetch_one_repo
+    one_repo_to_migration
+elif [ "${mode}" == "star" ]; then
+    log "Mode = starred repos"
+    set_uid
+    fetch_starred_repos
+    repos_to_migration
+elif [ "${mode}" == "user" ]; then
+    log "Mode = user"
+    set_uid_user
+    fetch_user_repos
+    repos_to_migration
+fi
+
+log "Finished."