Month: May 2023

So, you have a large pile of Bitbucket repos that you want to check out, mirror or even back up?

Here’s a shellscript for you that does the trick.

Prerequisites:

• jq, bash and git installed on the machine.
• You have an APP password on the bitbucket (create in personal settings)
• You have an SSH key on the bitbucket and you can checkout via ssh.

Store the script as something like “bb” and issue “chmod 755 bb”

There are 3 commands:
“bb  get” will fetch the lists of the repos.
“bb update” will mass check out the repos if it doesn’t exist locally, or update them.
“bb backup” will check out the repos in mirror format, and make a tgz file of it all.

The checkouts will be done under the path ./<org>/<project>/<repo>

Enjoy!

#!/bin/env bash
# ###########################################################################
# (C) EmberLabs / Chris Sprucefield. 
# Licensed under CC BY | https://creativecommons.org/licenses/by/4.0/
# Basics
ORG=""  # Fill in your org name here. This is the workspace name. 
USER="" # Fill in your username here.
PASS="" # Fill in your BB APP password here.

function getRepoLists() {
    AUTH="Basic $(echo -ne "${USER}:${PASS}" | base64)"
    PAGE="1"

    echo "--------------------------------------------------------------------"
    echo "Getting the ${ORG} repository lists from BitBucket"
    BURL="https://api.bitbucket.org/2.0/repositories/${ORG}?pagelen=100"
    GO="1"
    rm -f repolist-*.json
    echo -n "Getting page "
    while [ "$GO" != "0" ]
    do
      echo -n "${PAGE} "
      curl -s -H "Accept: application/json" -H "Authorization: ${AUTH}" "${BURL}&page=${PAGE}" -o "repolist-${PAGE}.json"

      if [ "$(grep "\"next\":" repolist-${PAGE}.json)" == "" ]
      then
        GO="0"
      fi

      let PAGE=$PAGE+1

    done
    echo ""
}

function cloneRefreshRepos() {
    CWD="$(pwd)"

    for list in $(ls repolist-*.json)
    do
      echo "Processing list $list"
      O="0"
      CONT="1"
      while [ "${CONT}" == "1" ]
      do
        cd "${CWD}"
        OBJECT="$(jq ".values[${O}]" "${CWD}/$list")"

        if [ "${OBJECT}" != "null" ]
        then
          PROJECT="$(echo "$OBJECT" | jq -r '.project.name')"
          NAME="$(echo "$OBJECT" | jq -r '.name')"
          SLUG="$(echo "$OBJECT" | jq -r '.slug')"
          CLONE="$(echo "$OBJECT" | jq -r '.links.clone[1].href')"

          if [ "$1" == "mirror" ]
          then
            mkdir -p "${ORG}-Mirror/${PROJECT}"
            cd "${ORG}-Mirror/${PROJECT}"
            if [ ! -e "${SLUG}" ]
            then
              echo "----------------------------------------------------------"
              git clone --mirror "${CLONE}"
              echo ""
            fi
          else
            mkdir -p "${ORG}/${PROJECT}"
            cd "${ORG}/${PROJECT}"
            if [ ! -e "${SLUG}" ]
            then
              echo "----------------------------------------------------------"
              echo "Cloning ${NAME}"
              git clone "${CLONE}"
              echo ""
            else
              cd "${SLUG}"
              echo "----------------------------------------------------------"
              echo "Updating ${NAME}"
              git fetch --all
              git pull
              echo ""
            fi
          fi

        else
          CONT="0"
        fi

        let O=$O+1
      done
    done
    cs ${CWD}
}


case $1 in
  get)
    # ########################################################################
    # Get the repo lists
    getRepoLists
    ;;

  update)
    # ########################################################################
    # Process the lists.
    cloneRefreshRepos
    ;;

  backup)
    # ########################################################################
    # Process the lists.
    cloneRefreshRepos mirror
    tar -zcf "${ORG}-Mirror-$(date -I).tgz" "${ORG}-Mirror"
    rm -fr "${ORG}-Mirror"
    ;;

  *)
    echo "Usage: "
    echo "<cmd>> get            Get the repo lists"
    echo "<cmd>> update         Update / checkout repos from BB"
    echo "<cmd>> backup         Create a mirror mackup of BB"
esac