Self-managed GitLab

#!/usr/bin/env bash

## gitlab-webhook.sh

# Add a webhook to one project, or every project in a subgroup tree

set -euo pipefail

migrate() (
  set +x
  umask 077
  local host='' project='' group='' from='' to='' report='' secret_map=''
  local secret_env=GITLAB_WEBHOOK_SECRET auth=private-token subgroups=false
  local apply=false test_event=false explicit_dry_run=false
  fail() { printf '[ERROR] %s\n' "$1" >&2; exit 1; }
  while (( $# )); do
    case "$1" in
      --help)
        printf '%s\n' \
          'Usage: gitlab-webhook.sh migrate --host HTTPS_ORIGIN (--project ID_OR_PATH | --group ID_OR_PATH)' \
          '  --from-url OLD_URL --to-url HTTPS_URL --report NEW_JSONL_FILE' \
          '  [--include-subgroups] [--dry-run | --apply] [--test]' \
          '  [--secret-env NAME | --secret-map JSON_FILE] [--auth private-token|bearer]' \
          'Set GITLAB_TOKEN; apply also requires the destination secret (default: GITLAB_WEBHOOK_SECRET).'
        exit 0 ;;
      --apply) apply=true; shift ;;
      --dry-run) explicit_dry_run=true; shift ;;
      --include-subgroups) subgroups=true; shift ;;
      --test) test_event=true; shift ;;
      --host|--project|--group|--from-url|--to-url|--report|--secret-env|--secret-map|--auth)
        (( $# >= 2 )) && [[ -n $2 && $2 != --* ]] || fail 'Missing option value'
        case "$1" in
          --host) host=$2 ;; --project) project=$2 ;; --group) group=$2 ;;
          --from-url) from=$2 ;; --to-url) to=$2 ;; --report) report=$2 ;;
          --secret-env) secret_env=$2 ;; --secret-map) secret_map=$2 ;; --auth) auth=$2 ;;
        esac
        shift 2 ;;
      *) fail 'Unknown migration option; use migrate --help' ;;
    esac
  done
  command -v jq >/dev/null || fail 'jq is required'
  [[ -n $host && -n $from && -n $to && -n $report ]] || fail 'Specify host, from-url, to-url and report'
  [[ -n $project && -z $group || -z $project && -n $group ]] || fail 'Choose exactly one project or group'
  [[ $subgroups == false || -n $group ]] || fail 'include-subgroups requires a group'
  [[ $apply == false || $explicit_dry_run == false ]] || fail 'Choose apply or dry-run, not both'
  [[ $test_event == false || $apply == true ]] || fail 'test requires apply and can trigger reviews'
  [[ $host =~ ^https://[^/?#@[:space:]]+/?$ ]] || fail 'Use an HTTPS GitLab origin without credentials'
  [[ $from =~ ^https?://[^/?#@[:space:]]+(/[^[:space:]#]*)?$ ]] || fail 'Invalid old webhook URL'
  [[ $to =~ ^https://[^/?#@[:space:]]+(/[^[:space:]#]*)?$ && $to != "$from" ]] || fail 'Use a different HTTPS destination without credentials'
  [[ -n ${GITLAB_TOKEN:-} && $GITLAB_TOKEN != *$'\n'* && $GITLAB_TOKEN != *$'\r'* ]] || fail 'Set GITLAB_TOKEN to an API token'
  [[ $auth == private-token || $auth == bearer ]] || fail 'auth must be private-token or bearer'
  if [[ -n $secret_map ]]; then
    jq -e 'type == "object" and all(.[]; type == "string")' "$secret_map" >/dev/null || fail 'Invalid secret map'
  fi
  local tmp api="${host%/}/api/v4"
  tmp=$(mktemp -d)
  trap 'rm -f "$tmp/auth" "$tmp/response" "$tmp/plan"; rmdir "$tmp"' EXIT
  if [[ $auth == bearer ]]; then
    printf 'Authorization: Bearer %s\n' "$GITLAB_TOKEN" > "$tmp/auth"
  else
    printf 'PRIVATE-TOKEN: %s\n' "$GITLAB_TOKEN" > "$tmp/auth"
  fi
  set -C
  exec 3> "$report" || fail 'Report must be a new writable file'
  set +C

  request() {
    local method=$1 path=$2 status attempt
    local body=()
    [[ $method != PUT ]] || body=(--header 'Content-Type: application/json' --data-binary @-)
    for attempt in 1 2 3; do
      status=$(curl --disable --silent --proto '=https' --connect-timeout 10 --max-time 60 \
        --header "@$tmp/auth" --request "$method" "${body[@]}" \
        --output "$tmp/response" --write-out '%{http_code}' "$api$path") || return 1
      if [[ $status == 2?? ]]; then
        jq -e . "$tmp/response"
        return
      fi
      [[ $method == GET && $status == 429 && $attempt -lt 3 ]] || return 1
      sleep "$((attempt * 2))"
    done
  }
  encode() { jq -rn --arg value "$1" '$value | @uri'; }
  list_all() {
    local path=$1 page=1 separator='?' batch count
    [[ $path != *'?'* ]] || separator='&'
    while :; do
      batch=$(request GET "$path${separator}per_page=100&page=$page") || return 1
      jq -e 'type == "array"' <<< "$batch" >/dev/null || return 1
      jq -c '.[]' <<< "$batch"
      count=$(jq length <<< "$batch")
      (( count == 100 )) || break
      page=$((page + 1))
    done
  }
  snapshot() {
    jq -cS 'with_entries(select(.key | test("^(id|url|.*_events|push_events_branch_filter|branch_filter_strategy|enable_ssl_verification)$")))'
  }
  report_row() {
    local result
    result=$(jq -cn --argjson project_id "$pid" --arg status "$1" --argjson hook "$2" \
      --arg destination "$to" '{project_id:$project_id,status:$status,hook:$hook,destination:$destination}')
    printf '%s\n' "$result" >&3
    printf '%s\n' "$result"
  }
  resolve_secret() {
    local name=$secret_env
    if [[ -n $secret_map ]]; then
      name=$(jq -r --arg pid "$pid" '.[$pid] // empty' "$secret_map") || return 1
    fi
    [[ $name =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || return 1
    export MIGRATION_WEBHOOK_SECRET="${!name:-}"
    [[ -n $MIGRATION_WEBHOOK_SECRET && $MIGRATION_WEBHOOK_SECRET != *$'\n'* && $MIGRATION_WEBHOOK_SECRET != *$'\r'* ]]
  }

  local projects pid hooks old target hook state failed=0
  if [[ -n $project ]]; then
    projects=$(request GET "/projects/$(encode "$project")") || fail 'Cannot read project'
  else
    projects=$(list_all "/groups/$(encode "$group")/projects?include_subgroups=$subgroups&with_shared=false") || fail 'Cannot list all group projects'
  fi
  projects=$(jq -sc 'if all(.[]; (.id | type == "number") and (.id > 0)) then . else error("Invalid projects") end' <<< "$projects") || fail 'Invalid project response'
  [[ $(jq length <<< "$projects") -gt 0 ]] || fail 'No accessible projects found; no updates applied'
  : > "$tmp/plan"
  while IFS= read -r pid; do
    hooks=$(list_all "/projects/$pid/hooks") || fail 'Cannot list all project webhooks; no updates applied'
    hooks=$(jq -s . <<< "$hooks")
    jq -e 'all(.[]; (.id | type == "number") and (.url | type == "string"))' <<< "$hooks" >/dev/null || fail 'Invalid webhook response'
    old=$(jq -c --arg url "$from" '[.[] | select(.url == $url)]' <<< "$hooks")
    target=$(jq -c --arg url "$to" '[.[] | select(.url == $url)]' <<< "$hooks")
    hook=null
    if [[ $(jq length <<< "$old") == 1 && $(jq length <<< "$target") == 0 ]]; then
      hook=$(jq '.[0]' <<< "$old" | snapshot)
      state=WOULD_UPDATE
      if [[ $apply == true ]] && ! resolve_secret; then state=MISSING_SECRET; failed=1; fi
    elif [[ $(jq length <<< "$old") == 0 && $(jq length <<< "$target") == 1 ]]; then
      state=ALREADY_TARGET_AUTH_UNVERIFIED
      hook=$(jq '.[0]' <<< "$target" | snapshot)
    elif [[ $(jq length <<< "$old") == 0 && $(jq length <<< "$target") == 0 ]]; then
      state=NO_MATCH
    else
      state=CONFLICT; failed=1
    fi
    report_row "$state" "$hook"
    if [[ $state == WOULD_UPDATE ]]; then
      jq -cn --argjson pid "$pid" --argjson hook "$hook" '{pid:$pid,hook:$hook}' >> "$tmp/plan"
    fi
  done < <(jq -r 'unique_by(.id)[].id' <<< "$projects")
  [[ $apply == true ]] || exit "$failed"
  (( failed == 0 )) || fail 'Resolve conflicts or missing secrets before applying; no updates applied'

  local row id current expected actual
  while IFS= read -r row; do
    pid=$(jq -r .pid <<< "$row")
    hook=$(jq -cS .hook <<< "$row")
    id=$(jq -r .id <<< "$hook")
    hooks=$(list_all "/projects/$pid/hooks") || { report_row READ_FAILED "$hook"; exit 1; }
    current=$(jq -sc --argjson id "$id" --arg from "$from" --arg to "$to" \
      '[.[] | select(.url == $from or .url == $to or .id == $id)]' <<< "$hooks")
    if [[ $(jq length <<< "$current") != 1 ]] || [[ $(jq '.[0]' <<< "$current" | snapshot) != "$hook" ]]; then
      report_row CHANGED_SINCE_PLAN "$hook"; exit 1
    fi
    resolve_secret || { report_row MISSING_SECRET "$hook"; exit 1; }
    report_row UPDATING "$hook"
    if ! jq -cn --arg url "$to" '{url:$url,token:env.MIGRATION_WEBHOOK_SECRET}' | request PUT "/projects/$pid/hooks/$id" >/dev/null; then
      report_row UPDATE_OUTCOME_UNKNOWN "$hook"; exit 1
    fi
    current=$(request GET "/projects/$pid/hooks/$id") || { report_row VERIFY_FAILED "$hook"; exit 1; }
    expected=$(jq -cS --arg url "$to" '.url=$url' <<< "$hook")
    actual=$(snapshot <<< "$current")
    if [[ $actual != "$expected" ]] || jq -e '.token_present == false' <<< "$current" >/dev/null; then
      report_row VERIFY_FAILED "$hook"; exit 1
    fi
    report_row UPDATED_AUTH_UNVERIFIED "$actual"
    if [[ $test_event == true ]]; then
      if request POST "/projects/$pid/hooks/$id/test/merge_requests_events" >/dev/null; then
        report_row TEST_REQUESTED_DELIVERY_UNVERIFIED "$actual"
      else
        report_row TEST_REQUEST_FAILED "$actual"; exit 1
      fi
    fi
  done < "$tmp/plan"
)

if [[ ${1:-} == migrate ]]; then
  shift
  migrate "$@"
  exit
fi

usage() {
  cat <<EOF
Usage:
  $0 -h <gitlab-host> -u <webhook-url> -s <webhook-secret> \\
     [-t <access-token>] [-A <auth-header>] [-p <project> | -g <group>] [-v]

Required:
  -h  GitLab host (e.g. gitlab.example.com)
  -u  Webhook endpoint URL to receive POSTs
  -s  Webhook secret token (used for signature verification)

Authentication (one of):
  -t  Access token (PAT, project, group or OAuth). If omitted, \$GITLAB_TOKEN is used
  -A  Auth header to use. Default detects:
        PAT → "PRIVATE-TOKEN"
        anything else → "Authorization: Bearer"

Scope (choose one):
  -p  Project ID or full path (e.g. 42 or group/app)
  -g  Group ID or full path, recurse through all subgroups & projects

Options:
  -v  Verbose output (show individual project IDs in final summary)
EOF
  exit 1
}

HOST="" HOOK_URL="" HOOK_SECRET=""
TOKEN="${GITLAB_TOKEN:-}" AUTH_HEADER=""
PROJECT="" GROUP="" VERBOSE=false

while getopts "h:u:s:t:A:p:g:v" opt; do
  case "$opt" in
    h) HOST=$OPTARG ;;
    u) HOOK_URL=$OPTARG ;;
    s) HOOK_SECRET=$OPTARG ;;
    t) TOKEN=$OPTARG ;;
    A) AUTH_HEADER=$OPTARG ;;
    p) PROJECT=$OPTARG ;;
    g) GROUP=$OPTARG ;;
    v) VERBOSE=true ;;
    *) usage ;;
  esac
done

# Mandatory checks

[[ -z $HOST || -z $HOOK_URL || -z $HOOK_SECRET ]] && usage
[[ -n $PROJECT && -n $GROUP ]] && usage
[[ -z $PROJECT && -z $GROUP ]] && usage

# Token handling

if [[ -z $TOKEN ]]; then
  echo "[ERROR] No access token provided. Use -t or set \$GITLAB_TOKEN" >&2
  exit 1
fi

# Choose header if not forced

if [[ -z $AUTH_HEADER ]]; then
  if [[ $TOKEN == glpat-* || $TOKEN == "PAT-"* ]]; then
    AUTH_HEADER="PRIVATE-TOKEN"
  else
    AUTH_HEADER="Authorization: Bearer"
  fi
fi

API="https://${HOST}/api/v4"
CURL_BASE=(curl -sSf --header "${AUTH_HEADER}: ${TOKEN}")

# Track processed projects to avoid duplicates

declare -A PROCESSED_PROJECTS
WEBHOOK_PROJECTS=()
EXISTING_WEBHOOK_PROJECTS=()
TOTAL_PROJECTS_FOUND=0
PROJECTS_PROCESSED=0

url_encode() {
  local string="$1"
  printf '%s' "$string" | sed 's/\//%2F/g; s/ /%20/g; s/@/%40/g; s/:/%3A/g; s/#/%23/g; s/?/%3F/g; s/&/%26/g; s/=/%3D/g; s/+/%2B/g'
}

fetch_paginated() {
  local url=$1
  local page=1
  local per_page=100

  while true; do
    local paginated_url
    if [[ "$url" == *"?"* ]]; then
      paginated_url="${url}&per_page=${per_page}&page=${page}"
    else
      paginated_url="${url}?per_page=${per_page}&page=${page}"
    fi

    local response
    response=$("${CURL_BASE[@]}" "$paginated_url" 2>/dev/null) || {
      echo "[ERROR] Failed to fetch page $page from $url" >&2
      return 1
    }

    if [[ "$response" == "[]" || "$response" == "null" ]]; then
      break
    fi

    local page_results
    page_results=$(echo "$response" | jq -r '.[].id' 2>/dev/null) || {
      echo "[ERROR] Failed to parse JSON response from page $page" >&2
      return 1
    }

    if [[ -z "$page_results" ]]; then
      break
    fi

    local page_count
    page_count=$(echo "$page_results" | wc -l)
    TOTAL_PROJECTS_FOUND=$((TOTAL_PROJECTS_FOUND + page_count))
    echo "[PROGRESS] Found $page_count projects on page $page (total: $TOTAL_PROJECTS_FOUND)" >&2

    echo "$page_results"

    local item_count
    item_count=$(echo "$response" | jq '. | length' 2>/dev/null) || 0
    if [[ "$item_count" -lt "$per_page" ]]; then
      break
    fi

    ((page++))
  done
}

create_hook() {
  local pid=$1

  if [[ -n "${PROCESSED_PROJECTS[$pid]:-}" ]]; then
    return 0
  fi

  PROCESSED_PROJECTS[$pid]=1
  PROJECTS_PROCESSED=$((PROJECTS_PROCESSED + 1))

  local encoded_pid
  if [[ $pid =~ ^[0-9]+$ ]]; then
    encoded_pid=$pid
  else
    encoded_pid=$(url_encode "$pid")
  fi

  local existing_webhooks
  existing_webhooks=$("${CURL_BASE[@]}" "${API}/projects/${encoded_pid}/hooks" 2>/dev/null) || {
    echo "[ERROR] Failed to fetch existing webhooks for project $pid" >&2
    return 1
  }

  if echo "$existing_webhooks" | jq -e --arg url "$HOOK_URL" '.[] | select(.url == $url)' >/dev/null 2>&1; then
    [[ "$VERBOSE" == "true" ]] && echo "[INFO] Webhook already exists for project: $pid" >&2
    EXISTING_WEBHOOK_PROJECTS+=("$pid")
    return 0
  fi

  [[ "$VERBOSE" == "true" ]] && echo "[INFO] Adding webhook to project: $pid" >&2

  "${CURL_BASE[@]}" --request POST \
    --data-urlencode "url=${HOOK_URL}" \
    --data "token=${HOOK_SECRET}" \
    --data "push_events=true" \
    --data "note_events=true" \
    --data "issues_events=true" \
    --data "merge_requests_events=true" \
    --data "enable_ssl_verification=true" \
    "${API}/projects/${encoded_pid}/hooks" \
    > /dev/null

  WEBHOOK_PROJECTS+=("$pid")
}

traverse_group() {
  local gid=$1
  local encoded_gid
  if [[ $gid =~ ^[0-9]+$ ]]; then
    encoded_gid=$gid
  else
    encoded_gid=$(url_encode "$gid")
  fi

  while IFS= read -r pid; do
    [[ -n "$pid" ]] && create_hook "$pid"
  done < <(
    fetch_paginated "${API}/groups/${encoded_gid}/projects?include_subgroups=true"
  )

  while IFS= read -r sg; do
    [[ -n "$sg" ]] && traverse_group "$sg"
  done < <(
    fetch_paginated "${API}/groups/${encoded_gid}/subgroups"
  )
}

echo "[INFO] Starting webhook processing..." >&2

if [[ -n $PROJECT ]]; then
  echo "[INFO] Processing single project: $PROJECT" >&2
  create_hook "$PROJECT"
else
  echo "[INFO] Processing group and subgroups: $GROUP" >&2
  traverse_group "$GROUP"
fi

echo "[INFO] Finished processing all projects" >&2

total_projects=$((${#WEBHOOK_PROJECTS[@]} + ${#EXISTING_WEBHOOK_PROJECTS[@]}))

if [[ $total_projects -eq 0 ]]; then
  echo "[INFO] No projects were processed"
else
  if [[ ${#WEBHOOK_PROJECTS[@]} -gt 0 ]]; then
    if [[ "$VERBOSE" == "true" ]]; then
      echo "[INFO] Webhooks installed successfully on ${#WEBHOOK_PROJECTS[@]} project(s):"
      for pid in "${WEBHOOK_PROJECTS[@]}"; do
        echo "  - Project ID: $pid"
      done
    else
      echo "[INFO] Webhooks installed successfully on ${#WEBHOOK_PROJECTS[@]} project(s)"
    fi
  fi

  if [[ ${#EXISTING_WEBHOOK_PROJECTS[@]} -gt 0 ]]; then
    if [[ "$VERBOSE" == "true" ]]; then
      echo "[INFO] Webhooks already existed on ${#EXISTING_WEBHOOK_PROJECTS[@]} project(s):"
      for pid in "${EXISTING_WEBHOOK_PROJECTS[@]}"; do
        echo "  - Project ID: $pid"
      done
    else
      echo "[INFO] Webhooks already existed on ${#EXISTING_WEBHOOK_PROJECTS[@]} project(s)"
    fi
  fi

  echo "[INFO] Total projects processed: $total_projects"
fi