Currently using Sonarqube 8.3.1.34397 Community Edition.
Installed Sonarqube and Postgres on the same AWS instance, everything is working fine. When want to change the Postgres connection to RDS instance then unable to start the Sonarqube service.
Below is the error from the es.log
2020.06.19 06:35:26 WARN es[o.e.g.G.InternalPrimaryShardAllocator] [views][0]: failed to list shard for shard_started on node [0nBtsQ_9SZeB5AGDa56XTg]
org.elasticsearch.action.FailedNodeException: Failed node [0nBtsQ_9SZeB5AGDa56XTg]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:236) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$200(TransportNodesAction.java:151) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:210) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1114) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.processException(TransportService.java:1226) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$DirectResponseChannel.sendResponse(TransportService.java:1200) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$7.onFailure(TransportService.java:703) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.onFailure(ThreadContext.java:736) [elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) [elasticsearch-6.8.4.jar:6.8.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.elasticsearch.transport.RemoteTransportException: [sonarqube][127.0.0.1:9001][internal:gateway/local/started_shards[n]]
Caused by: org.elasticsearch.ElasticsearchException: failed to load started shards
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:169) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:61) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:692) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.4.jar:6.8.4]
… 3 more
Caused by: org.elasticsearch.ElasticsearchException: java.io.IOException: failed to read [id:171, file:/opt/sonar/sonarqube-8.3.1.34397/data/es6/nodes/0/indices/KZBpCC1FQRuNBBpX6-yHPg/_state/state-171.st]
at org.elasticsearch.ExceptionsHelper.maybeThrowRuntimeAndSuppress(ExceptionsHelper.java:165) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestStateWithGeneration(MetaDataStateFormat.java:306) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:324) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:127) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:61) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:692) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.4.jar:6.8.4]
… 3 more
Caused by: java.io.IOException: failed to read [id:171, file:/opt/sonar/sonarqube-8.3.1.34397/data/es6/nodes/0/indices/KZBpCC1FQRuNBBpX6-yHPg/_state/state-171.st]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestStateWithGeneration(MetaDataStateFormat.java:300) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:324) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:127) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:61) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:692) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.4.jar:6.8.4]
… 3 more
Caused by: java.nio.file.NoSuchFileException: /opt/sonar/sonarqube-8.3.1.34397/data/es6/nodes/0/indices/KZBpCC1FQRuNBBpX6-yHPg/_state/state-171.st
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:219) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:370) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:421) ~[?:?]
at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77) ~[lucene-core-7.7.2.jar:7.7.2 d4c30fc2856154f2c1fefc589eb7cd070a415b94 - janhoy - 2019-05-28 23:30:25]
at org.elasticsearch.gateway.MetaDataStateFormat.read(MetaDataStateFormat.java:183) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestStateWithGeneration(MetaDataStateFormat.java:296) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.MetaDataStateFormat.loadLatestState(MetaDataStateFormat.java:324) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:127) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.gateway.TransportNodesListGatewayStartedShards.nodeOperation(TransportNodesListGatewayStartedShards.java:61) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:692) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-6.8.4.jar:6.8.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-6.8.4.jar:6.8.4]
… 3 more
2020.06.19 06:35:27 INFO es[o.e.g.GatewayService] recovered [7] indices into cluster_state
2020.06.19 06:35:28 INFO es[o.e.c.r.a.AllocationService] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[metadatas][0]] …]).
2020.06.19 06:35:30 INFO es[o.e.n.Node] stopping …
2020.06.19 06:35:30 INFO es[o.e.n.Node] stopped
2020.06.19 06:35:30 INFO es[o.e.n.Node] closing …
2020.06.19 06:35:30 INFO es[o.e.n.Node] closed
Below is the error from the sonar.log
–> Wrapper Started as Daemon
Launching a JVM…
Wrapper (Version 3.2.3) http://wrapper.tanukisoftware.org
Copyright 1999-2006 Tanuki Software, Inc. All Rights Reserved.
2020.06.19 06:35:11 INFO app[o.s.a.AppFileSystem] Cleaning or creating temp directory /opt/sonar/sonarqube-8.3.1.34397/temp
2020.06.19 06:35:11 INFO app[o.s.a.es.EsSettings] Elasticsearch listening on /127.0.0.1:9001
2020.06.19 06:35:11 INFO app[o.s.a.ProcessLauncherImpl] Launch process[[key=‘es’, ipcIndex=1, logFilenamePrefix=es]] from [/opt/sonar/sonarqube-8.3.1.34397/elasticsearch]: /opt/sonar/sonarqube-8.3.1.34397/elasticsearch/bin/elasticsearch
2020.06.19 06:35:11 INFO app[o.s.a.SchedulerImpl] Waiting for Elasticsearch to be up and running
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
2020.06.19 06:35:12 INFO app[o.e.p.PluginsService] no modules loaded
2020.06.19 06:35:12 INFO app[o.e.p.PluginsService] loaded plugin [org.elasticsearch.transport.Netty4Plugin]
2020.06.19 06:35:29 INFO app[o.s.a.SchedulerImpl] Process[es] is up
2020.06.19 06:35:29 INFO app[o.s.a.ProcessLauncherImpl] Launch process[[key=‘web’, ipcIndex=2, logFilenamePrefix=web]] from [/opt/sonar/sonarqube-8.3.1.34397]: /usr/lib/jvm/java-11-openjdk-amd64/bin/java -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/opt/sonar/sonarqube-8.3.1.34397/temp -XX:-OmitStackTraceInFastThrow --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.rmi/sun.rmi.transport=ALL-UNNAMED -Xmx512m -Xms128m -XX:+HeapDumpOnOutOfMemoryError -Dhttp.proxyHost=127.0.0.1 -Dhttp.proxyPort=80 -Dhttp.nonProxyHosts=localhost|127.|[::1] -Dhttps.proxyHost=127.0.0.1 -Dhttps.proxyPort=80 -cp ./lib/common/:/opt/sonar/sonarqube-8.3.1.34397/lib/jdbc/postgresql/postgresql-42.2.11.jar org.sonar.server.app.WebServer /opt/sonar/sonarqube-8.3.1.34397/temp/sq-process13669484798542940787properties
2020.06.19 06:35:30 INFO app[o.s.a.SchedulerImpl] Process[web] is stopped
2020.06.19 06:35:31 WARN app[o.s.a.p.AbstractManagedProcess] Process exited with exit value [es]: 143
2020.06.19 06:35:31 INFO app[o.s.a.SchedulerImpl] Process[es] is stopped
2020.06.19 06:35:31 INFO app[o.s.a.SchedulerImpl] SonarQube is stopped
<-- Wrapper Stopped
Below is the error from web.log:
2020.06.19 06:35:30 ERROR web[o.s.s.a.EmbeddedTomcat] Fail to start web server
org.apache.catalina.LifecycleException: Failed to initialize connector [Connector[HTTP/1.1-5432]]
at org.apache.catalina.core.StandardService.initInternal(StandardService.java:559)
at org.apache.catalina.util.LifecycleBase.init(LifecycleBase.java:136)
at org.apache.catalina.core.StandardServer.initInternal(StandardServer.java:848)
at org.apache.catalina.util.LifecycleBase.init(LifecycleBase.java:136)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:173)
at org.apache.catalina.startup.Tomcat.start(Tomcat.java:440)
at org.sonar.server.app.EmbeddedTomcat.start(EmbeddedTomcat.java:65)
at org.sonar.server.app.WebServer.start(WebServer.java:52)
at org.sonar.process.ProcessEntryPoint.launch(ProcessEntryPoint.java:97)
at org.sonar.process.ProcessEntryPoint.launch(ProcessEntryPoint.java:81)
at org.sonar.server.app.WebServer.main(WebServer.java:99)
2020.06.19 06:35:30 WARN web[o.s.p.ProcessEntryPoint] Fail to start web
java.lang.RuntimeException: org.apache.catalina.LifecycleException: Failed to initialize connector [Connector[HTTP/1.1-5432]]
at com.google.common.base.Throwables.propagate(Throwables.java:241)
at org.sonar.server.app.EmbeddedTomcat.start(EmbeddedTomcat.java:69)
at org.sonar.server.app.WebServer.start(WebServer.java:52)
at org.sonar.process.ProcessEntryPoint.launch(ProcessEntryPoint.java:97)
at org.sonar.process.ProcessEntryPoint.launch(ProcessEntryPoint.java:81)
at org.sonar.server.app.WebServer.main(WebServer.java:99)
Caused by: org.apache.catalina.LifecycleException: Failed to initialize connector [Connector[HTTP/1.1-5432]]
at org.apache.catalina.core.StandardService.initInternal(StandardService.java:559)
at org.apache.catalina.util.LifecycleBase.init(LifecycleBase.java:136)
at org.apache.catalina.core.StandardServer.initInternal(StandardServer.java:848)
at org.apache.catalina.util.LifecycleBase.init(LifecycleBase.java:136)
at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:173)
at org.apache.catalina.startup.Tomcat.start(Tomcat.java:440)
at org.sonar.server.app.EmbeddedTomcat.start(EmbeddedTomcat.java:65)
… 4 common frames omitted
2020.06.19 06:35:30 INFO web[o.s.p.ProcessEntryPoint] Hard stopping process
Appreciate your help and thoughts.