SonarQube Server Health probe null pointer exception

Everyday when our SonarQube Server(developer-8.9.2-master-10196) pod gets redeployed the SonarQube instance is giving the following error:

2021.09.23 20:02:50 ERROR web[AXwUCQ5SIPNQAqHdAAhP][o.s.s.w.WebServiceEngine] Fail to process request http://localhost:9000/api/system/health
998
java.lang.NullPointerException: status can't be null
997
	at java.base/java.util.Objects.requireNonNull(Unknown Source)
996
	at org.sonar.server.health.Health$Builder.checkStatus(Health.java:118)
995
	at org.sonar.server.health.Health$Builder.build(Health.java:113)
994
	at org.sonar.server.health.HealthReducer.apply(HealthReducer.java:45)
993
	at org.sonar.server.health.HealthReducer.apply(HealthReducer.java:27)
992
	at java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)
991
	at java.base/java.util.stream.ReferencePipeline$3$1.accept(Unknown Source)
990
	at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)
989
	at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
988
	at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
987
	at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)
986
	at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
985
	at java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)
984
	at org.sonar.server.health.HealthCheckerImpl.checkNode(HealthCheckerImpl.java:66)
983
	at org.sonar.server.platform.ws.HealthActionSupport.checkNodeHealth(HealthActionSupport.java:62)
982
	at org.sonar.server.platform.ws.HealthAction.handle(HealthAction.java:56)
981
	at org.sonar.server.ws.WebServiceEngine.execute(WebServiceEngine.java:110)
980
	at org.sonar.server.platform.web.WebServiceFilter.doFilter(WebServiceFilter.java:84)
979
	at org.sonar.server.platform.web.MasterServletFilter$GodFilterChain.doFilter(MasterServletFilter.java:139)
978
	at org.sonar.server.platform.web.SonarLintConnectionFilter.doFilter(SonarLintConnectionFilter.java:66)
977
	at org.sonar.server.platform.web.MasterServletFilter$GodFilterChain.doFilter(MasterServletFilter.java:139)
976
	at org.sonar.server.platform.web.MasterServletFilter.doFilter(MasterServletFilter.java:108)
975
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
974
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
973
	at org.sonar.server.platform.web.UserSessionFilter.doFilter(UserSessionFilter.java:81)
972
	at org.sonar.server.platform.web.UserSessionFilter.doFilter(UserSessionFilter.java:68)
971
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
970
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
969
	at org.sonar.server.platform.web.CacheControlFilter.doFilter(CacheControlFilter.java:76)
968
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
967
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
966
	at org.sonar.server.platform.web.SecurityServletFilter.doHttpFilter(SecurityServletFilter.java:76)
965
	at org.sonar.server.platform.web.SecurityServletFilter.doFilter(SecurityServletFilter.java:48)
964
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
963
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
962
	at org.sonar.server.platform.web.RedirectFilter.doFilter(RedirectFilter.java:58)
961
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
960
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
959
	at org.sonar.server.platform.web.RequestIdFilter.doFilter(RequestIdFilter.java:66)
958
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
957
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
956
	at org.sonar.server.platform.web.RootFilter.doFilter(RootFilter.java:62)
955
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
954
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
953
	at org.apache.catalina.filters.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:109)
952
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:194)
951
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:167)
950
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
949
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
948
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:544)
947
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143)
946
	at ch.qos.logback.access.tomcat.LogbackValve.invoke(LogbackValve.java:256)
945
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
944
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
943
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:364)
942
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:624)
941
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
940
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:831)
939
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1651)
938
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
937
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
936
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
935
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
934
	at java.base/java.lang.Thread.run(Unknown Source)

This would be fine if its for 5 minutes but this error is being thrown for 9 hours for 18 times.
While the check is done every minute as configured below.

      livenessProbe:
        exec:
          command:
            - sh
            - '-c'
            - >-
              set -e; STATUS_CURL_COMMAND="curl --silent
              http://localhost:9000/api/system/status";
              STATUS_GREP_COMMAND="grep --quiet --perl-regexp
              '^{.*\"status\"\:\"(UP|DB_MIGRATION_NEEDED|DB_MIGRATION_RUNNING)\".*}$'";

              if $STATUS_CURL_COMMAND | eval $STATUS_GREP_COMMAND; then
                echo "Status check: OK";
              else
                echo "Status check: FAIL";
                exit 1;
              fi

              SONARQUBE_USERNAME=$(cat
              /run/secrets/sonarqube-localadmin-credentials/SONARQUBE_USERNAME
              2>/dev/null || :); SONARQUBE_PASSWORD=$(cat
              /run/secrets/sonarqube-localadmin-credentials/SONARQUBE_PASSWORD
              2>/dev/null || :); HEALTH_CURL_COMMAND="curl --silent
              http://localhost:9000/api/system/health";
              HEALTH_GREP_COMMAND="grep --quiet --perl-regexp
              '^{.*\"health\":\"(?:GREEN|YELLOW)\".*}$'";

              if $HEALTH_CURL_COMMAND -u $SONARQUBE_USERNAME:$SONARQUBE_PASSWORD
              | eval $HEALTH_GREP_COMMAND; then
                echo "Health check: OK with credentials from secret";
              elif $HEALTH_CURL_COMMAND -u admin:admin | eval
              $HEALTH_GREP_COMMAND; then
                echo "Health check: OK with default credentials";
              else
                echo "Health check: FAIL";
                exit 1;
              fi
        failureThreshold: 60
        initialDelaySeconds: 600
        periodSeconds: 60
        successThreshold: 1
        timeoutSeconds: 1
      name: sonarqube
      ports:
        - containerPort: 9000
          name: http
          protocol: TCP
      readinessProbe:
        exec:
          command:
            - sh
            - '-c'
            - >-
              set -e; STATUS_CURL_COMMAND="curl --silent
              http://localhost:9000/api/system/status";
              STATUS_GREP_COMMAND="grep --quiet --perl-regexp
              '^{.*\"status\"\:\"(UP|DB_MIGRATION_NEEDED|DB_MIGRATION_RUNNING)\".*}$'";

              if $STATUS_CURL_COMMAND | eval $STATUS_GREP_COMMAND; then
                echo "Status check: OK";
              else
                echo "Status check: FAIL";
                exit 1;
              fi

              SONARQUBE_USERNAME=$(cat
              /run/secrets/sonarqube-localadmin-credentials/SONARQUBE_USERNAME
              2>/dev/null || :); SONARQUBE_PASSWORD=$(cat
              /run/secrets/sonarqube-localadmin-credentials/SONARQUBE_PASSWORD
              2>/dev/null || :); HEALTH_CURL_COMMAND="curl --silent
              http://localhost:9000/api/system/health";
              HEALTH_GREP_COMMAND="grep --quiet --perl-regexp
              '^{.*\"health\":\"(?:GREEN|YELLOW)\".*}$'";

              if $HEALTH_CURL_COMMAND -u $SONARQUBE_USERNAME:$SONARQUBE_PASSWORD
              | eval $HEALTH_GREP_COMMAND; then
                echo "Health check: OK with credentials from secret";
              elif $HEALTH_CURL_COMMAND -u admin:admin | eval
              $HEALTH_GREP_COMMAND; then
                echo "Health check: OK with default credentials";
              else
                echo "Health check: FAIL";
                exit 1;
              fi
        failureThreshold: 1
        periodSeconds: 15
        successThreshold: 1
        timeoutSeconds: 1

What could be the cause of this error?

Hey @Merlijn_Vermeer ,

We have been able to reproduce this issue during load testing.

We think that this is only happening during high load, and have created a ticket to investigate it further.

Hey @Belen_Pruvost,

Thanks for looking into it maybe when this gets fixed it will fix my problem.
But this error doesn’t get thrown on high load but on redeployment.

1 Like

Thanks for the clarification here, I’ve made the fact that this is happening on redeployment explicit in our ticket as well.