Hi,
As announced above, here you can find the script i use as workaround for this issue:
#!/usr/bin/python2.7
# -*- coding: utf-8 -*-
#
# Purpose: Remove Sonarqube short-lived branches older than x days (30 by default) months. Sonarqube can be configured to remove short-lived branches if they are not analysed for 30 days (triggered by long-lived branch analysis).
# We have a bug in production that prevent auto remove, so here is a scripts that will be croned to remove via rest api old short-lived branches.
#
# Notes : Algorithm
# For all Sonarqube's projects
# for each project
# get list of short-lived branches
# for each short-lived branches
# if last_analysis_date > x days_before_purging
# remove it
# log-it
#
# Ressources:
# - Support ticket opened to Sonarsource Community : https://community.sonarsource.com/t/sonarqube-short-lived-branches-are-never-purged/24318
# - Sonarqube Rest API: https://your_sonarqube_instance/your_context_root/web_api , context can be sonar or nothing
# File History:
# 27-05-20 N. M. - Created file
#
# tested with python 2.7
#
# usage: python removeSonarqubeShortLivedBranches.py
#
# prerequisite
#pip install 'python-dateutil==1.5'
#************************************************************
#********************** BEGIN *******************************
#************************************************************
import ldap
import sys
import argparse
import ConfigParser
# importing the requests library for HTTP requests
import requests
import urllib3
urllib3.disable_warnings()
import os
import logging
import logging.handlers
# These two lines enable debugging at httplib level (requests->urllib3->http.client)
# You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
# The only thing missing will be the response.body which is not logged.
try:
import http.client as http_client
except ImportError:
# Python 2
import httplib as http_client
from datetime import datetime
from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
import locale
import codecs
import os.path
from os import path
########### Begin of function declaration
def get_session(sonarqube_login, sonarqube_password):
session =requests.Session()
session.auth = (sonarqube_login, sonarqube_password)
session.verify = False
return session
def delete_branch(session, sonarqube_url, branch_name, projects_key):
URL = sonarqube_url + "/sonar6/api/project_branches/delete"
# example:$ curl --data "name=SonarqubeShortLivedBranches&project=com.myorganisation.project.removeSonarqubeShortLivedBranches:create-project" https://your_sonarqube_instance/your_context_root/api/projects/create -k
r = session.post( URL, data = {'branch':branch_name , 'project':projects_key})
logging.info( "delete_project status code %s", r.status_code)
# r.status_code == requests.codes.ok
########### End of function declaration
#************************************************************
#********************** MAIN ********************************
#************************************************************
#configuration variables
sonarqube_url = "https://your_sonarqube_instance"
sonarqube_token = "sonarqube_token"
sonarqube_password = "" #set to empty as using token for authentication
sonarqube_log_folder = "/var/Sonarqube/my_script/" #for example
#vars
page_size = "500" #Note: maximum value of page_size is 500, we will have a problem if we have more than 500 projects as we don't know how to load them via api
debug_mode = False
start_time = datetime.now()
days_before_purging = 30
branch_type = "SHORT"
script_name = "removeSonarqubeShortLivedBranches"
log_filename= sonarqube_log_folder + script_name + '.log'
# log_filename= sonarqube_log_folder + script_name + '.log'
# To have a very detailed logging of HTTP Request, uncomment line above
# http_client.HTTPConnection.debuglevel = 1
#logging management
# Change root logger level from WARNING (default) to NOTSET in order for all messages to be delegated.
logging.getLogger().setLevel(logging.NOTSET)
# Add stdout handler, with level INFO
console = logging.StreamHandler(sys.stdout)
console.setLevel(logging.INFO)
formater = logging.Formatter('[%(levelname)s] - %(message)s')
console.setFormatter(formater)
logging.getLogger().addHandler(console)
# Add file rotating handler, with level DEBUG
# Check if log exists and should therefore be rolled
needRoll = os.path.isfile(log_filename)
rotatingHandler = logging.handlers.RotatingFileHandler(filename=log_filename, backupCount=5)
rotatingHandler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - ' + script_name + ' - %(levelname)s - %(message)s')
rotatingHandler.setFormatter(formatter)
logging.getLogger().addHandler(rotatingHandler)
if needRoll:
# Roll over on application start
rotatingHandler.doRollover()
logging.info("************************************************************")
logging.info("********************** BEGIN *******************************")
logging.info("Script is going to check all sonarqube project by key...")
# For all sonarqube projects
URL = sonarqube_url + "/your_context_root/api/projects/search?ps=" + page_size
# preparing session
session = get_session(sonarqube_token, sonarqube_password)
# sending GET request and saving the response as a response object
# HTTP Get request: #https://your_sonarqube_instance/your_context_root/api/projects/search?ps=500
r = session.get( url = URL)
# extracting data in json format
projects_list = r.json()
# for each project
number_of_projects = len(projects_list['components'])
num_of_removed_branches = 0 #used for statistics only
num_of_kept_branches = 0 #used for statistics only
for component_index in range(number_of_projects):
try:
projects_key = projects_list['components'][component_index]['key'].decode('utf8')
# get list of short-lived branches
# for each short-lived branches
URL = sonarqube_url + "/your_context_root/api/project_branches/list?project=" + projects_key
r = session.get(url = URL)
project_branches = r.json()
logging.info('Working on project key: %s', projects_key)
for branch_index in range(len(project_branches['branches'])):
try:
branch_analysis_date = parse(project_branches['branches'][branch_index]['analysisDate']).date()
branch_name = project_branches['branches'][branch_index]['name']
isBranchOfType = project_branches['branches'][branch_index]['type'] == branch_type
# if last_analysis_date > days_before_purging (30 for example)
# compare branch last analysis date with (today - days_before_purging), which is like to compare if branch is older than "days_before_purging"
# for example : does branch_analysis_date=2019-11-19T12:57:29+0000 < (today - 30)=2020-04-27 ?, it returns a boolean
isOld = branch_analysis_date < (datetime.today()+relativedelta(days = -days_before_purging)).date()
if isOld and isBranchOfType :
# remove it
delete_branch(session, sonarqube_url, branch_name, projects_key)
logging.info(' branch: %s analysed on %s has been removed', branch_name, branch_analysis_date)
num_of_removed_branches = num_of_removed_branches + 1
elif isBranchOfType :
logging.info(' keeping branch: %s analysed on %s' , branch_name, branch_analysis_date)
num_of_kept_branches += 1
except KeyError:
logging.warning(' key error: you tried to access to a dictionary key that doesn\'t exist, may be caused by a project that has no branches or no analysis date ')
except UnicodeEncodeError:
logging.error(" You have met an encoding error!!! Find below your system encoding config and what i usally like. Try with export PYTHONIOENCODING=UTF-8 in your environment")
logging.error( "sys.stdout.encoding: %s",sys.stdout.encoding)
logging.error( "is a tty: %s", sys.stdout.isatty())
logging.error( "locale.getpreferredencoding: %s", locale.getpreferredencoding())
logging.error( "sys.getfilesystemencoding: %s", sys.getfilesystemencoding())
logging.error( "os.environ[PYTHONIOENCODING]: %s", os.environ["PYTHONIOENCODING"])
#print "chr(246), chr(9786), chr(9787: ", chr(246), chr(9786), chr(9787)
logging.error(" should give ")
logging.error( " utf_8 ")
logging.error( " False ")
logging.error( " ANSI_X3.4-1968 ")
logging.error( " ascii ")
logging.error( " utf_8 ")
logging.error( " ö âș â» " )
logging.error( "Please note that there is a known bug in windows when using python 2.7 that won't be fixed: https://stackoverflow.com/questions/1910275/unicode-filenames-on-windows-with-python-subprocess-popen?noredirect=1&lq=1")
logging.error( "You can try to solve it by integratingatin win_subprocess.py ")
#Statistics
logging.info("************************************************************")
logging.info("********************** Statistics **************************")
logging.info("Number of projetcs: %d", number_of_projects)
logging.info("Number of branches removed: %d", num_of_removed_branches)
logging.info("Number of branches keept: %d", num_of_kept_branches)
logging.info(datetime.now() - start_time )
logging.info("************************************************************")
logging.info("********************** End *********************************")
Hopes that will help some.
Regards,
. We donât use P/Rs Sonarqube functionality as it is not compatible with Gitlab for this version of sonarqube.
