You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@cloudstack.apache.org by cr...@istream.today on 2021/09/23 09:31:08 UTC
Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Hello,
I'm not sure why I get this error (added new vmware zone), there are
not permission issues for systemvms folder, and I do not see any error above
this. If I try to login to login into SSVM/Proxy console via cloudstack
management, I get this access denied.
Any suggestion?
021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
(DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication SSH
user root on host 10.15.0.160
2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
(DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get message
for exception: Failed to authentication SSH user root on host 10.15.0.160
2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
(DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to Exception:
java.lang.Exception
Message: Failed to authentication SSH user root on host 10.15.0.160
2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
(DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution result:
false
2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
(DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
Response Received:
2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
(DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
Processing: { Ans: , MgmtId: 345049356158, via: 40(lnd-uk-001.shape.host),
Ver: v1, Flags: 10,
[{"org.apache.cloudstack.ca.SetupKeystoreAnswer":{"result":"true","wait":"0"
,"bypassHostMaintenance":"false"}}] }
2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
(Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
(logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: , MgmtId:
345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
SetupKeystoreAnswer } }
2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
(Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
(logid:f3cd2252) Retrying after catching exception while trying to secure
agent for systemvm id=1955
com.cloud.utils.exception.CloudRuntimeException: Unable to read/process CSR:
Command failed due to Exception: java.lang.Exception
Message: Failed to authentication SSH user root on host 10.15.0.160
at
org.apache.cloudstack.ca.provider.RootCAProvider.generateCertificateUsingCsr
(RootCAProvider.java:170)
at
org.apache.cloudstack.ca.provider.RootCAProvider.issueCertificate(RootCAProv
ider.java:230)
at
org.apache.cloudstack.ca.CAManagerImpl.issueCertificate(CAManagerImpl.java:1
64)
at jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
Source)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Delegatin
gMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopU
tils.java:344)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint
(ReflectiveMethodInvocation.java:198)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
iveMethodInvocation.java:163)
at
org.apache.cloudstack.network.contrail.management.EventUtils$EventIntercepto
r.invoke(EventUtils.java:107)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
iveMethodInvocation.java:175)
at
com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51
)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
iveMethodInvocation.java:175)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(Expos
eInvocationInterceptor.java:97)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
iveMethodInvocation.java:186)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopPro
xy.java:215)
at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
at
com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMachineMana
gerImpl.java:970)
at
com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManage
rImpl.java:1241)
at
com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManage
rImpl.java:5502)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodA
ccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Delegatin
gMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.jav
a:107)
at
com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManager
Impl.java:5669)
at
com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext
(AsyncJobManagerImpl.java:620)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedCo
ntextRunnable.java:48)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(Defa
ultManagedContext.java:55)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithCon
text(DefaultManagedContext.java:102)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithCont
ext(DefaultManagedContext.java:52)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedCont
extRunnable.java:45)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJob
ManagerImpl.java:568)
at
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java
:515)
at
java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecut
or.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecu
tor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Regards,
Cristian
Re: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Posted by Daan Hoogland <da...@gmail.com>.
Christian,
Yes KVM and VMWARE are very different, this is because the underlying os is
different.
On Wed, Sep 29, 2021 at 7:22 PM <cr...@istream.today> wrote:
> Hi,
>
> Yes, it is the server, but I already started to check the BIOS ...
> and I have very interesting question, why with KVM is working? What is OS
> different between KVM and VMware when comes to systemvm patching/inject
> keys? I think here I have the answer to may error.
>
> Regards,
> Cristian
>
> -----Original Message-----
> From: Daan Hoogland <da...@gmail.com>
> Sent: Tuesday, September 28, 2021 12:14 PM
> To: users <us...@cloudstack.apache.org>
> Subject: Re: Unable to read/process CSR: Command failed due to Exception:
> java.lang.Exception
>
> ok, than I would like to add an idea to your "it's the server". Not saying
> you are wrong but location points to network i.e. firewall(s).
> Hope you strike luck,
>
> On Tue, Sep 28, 2021 at 10:58 AM <cr...@istream.today> wrote:
>
> > Hi,
> >
> > We already tested in all the ways is possible, this problem is
> > related to these specific servers, 100%.
> >
> > The last test we did, was to add the ESXI server to the zone we
> > already have and works, we disabled all hosts from that zone and
> > destroyed the SSVM to force a redeploy on this ESXI, of course, we
> > have ended with the same error.
> >
> > I want to mention that we have the exact same ESXI version and
> > hardware, for me looks like there is a BIOS/HW/DISK issue.
> >
> >
> > FYI:
> >
> > "I did this in 3 different new Setup (new Management install, NFS,
> > etc), no matter if I add as first zone or second zone.
> >
> > " I have tested this with multiple Cloudstack versions, 4.15.0,
> > 4.15.1,
> > 4.15.2 and VMware, 6.5 and with 6.7 different patches and with the
> > last patches. I'm 100% that is related to these 2 servers, I do not
> > understand what is wrong with these servers, this is the problem.
> >
> > We have 2 identical servers' hardware/ESXI, the only difference is
> > the location, the not working, are in UK, and any other location we
> > have, DE, NYC, FR, CA, etc. works fine. But we tested with the exact
> > same servers which are from DE(Germany) works perfect, with any ACS or
> > VMware version (also same CIDR range/size for public and private),
> > when we test with UK, we get same error no matter what."
> >
> > I want to mention that we have these servers (UK, DE, FR, CA ) from
> > OVH in vRack and the UK, are not working. ( I manage these server
> > for more than 5 years )"
> >
> > Regards,
> > Cristian
> >
> > -----Original Message-----
> > From: Daan Hoogland <da...@gmail.com>
> > Sent: Tuesday, September 28, 2021 11:00 AM
> > To: users <us...@cloudstack.apache.org>
> > Subject: Re: Unable to read/process CSR: Command failed due to Exception:
> > java.lang.Exception
> >
> > Christian, did you solve this (i just encountered your mail and see it
> > is yet unanswered) the problem is with the certificate generation, it
> > seems to try and generate from an empty string (the csr/certificate
> > signing request) Not being able to sign in to the SVM is a problem but
> > on vmware you would use a specific command as described in [1]. Did
> > you try `ssh -i /opt/xensource/bin/id_rsa --p 3922 root@privateIP
> > <ro...@10.209.161.128> OfTheHost`?
> >
> > [1]
> >
> > https://cwiki.apache.org/confluence/display/CLOUDSTACK/SSVM%2C+templat
> > es%2C+Secondary+storage+troubleshooting
> >
> > On Thu, Sep 23, 2021 at 11:31 AM <cr...@istream.today> wrote:
> >
> > > Hello,
> > >
> > >
> > >
> > > I'm not sure why I get this error (added new vmware zone),
> > > there are not permission issues for systemvms folder, and I do not
> > > see any error above this. If I try to login to login into SSVM/Proxy
> > > console via cloudstack management, I get this access denied.
> > >
> > >
> > >
> > > Any suggestion?
> > >
> > >
> > >
> > >
> > >
> > >
> > >
> > >
> > >
> > > 021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
> > > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > > job-34950/job-34972,
> > > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication
> > > SSH user root on host 10.15.0.160
> > >
> > > 2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
> > > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > > job-34950/job-34972,
> > > cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get
> > > message for exception: Failed to authentication SSH user root on
> > > host
> > > 10.15.0.160
> > >
> > > 2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
> > > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > > job-34950/job-34972,
> > > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to
> > > Exception:
> > > java.lang.Exception
> > >
> > > Message: Failed to authentication SSH user root on host 10.15.0.160
> > >
> > >
> > >
> > > 2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
> > > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > > job-34950/job-34972,
> > > cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution
> > > result:
> > > false
> > >
> > > 2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
> > > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> > 40-1394708509601300577:
> > > Response Received:
> > >
> > > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> > 40-1394708509601300577:
> > > Processing: { Ans: , MgmtId: 345049356158, via:
> > > 40(lnd-uk-001.shape.host),
> > > Ver: v1, Flags: 10,
> > > [{"org.apache.cloudstack.ca
> > > .SetupKeystoreAnswer":{"result":"true","wait":"0"
> > > ,"bypassHostMaintenance":"false"}}] }
> > >
> > > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > > (logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: ,
> MgmtId:
> > > 345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
> > > SetupKeystoreAnswer } }
> > >
> > > 2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
> > > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > > (logid:f3cd2252) Retrying after catching exception while trying to
> > > secure agent for systemvm id=1955
> > >
> > > com.cloud.utils.exception.CloudRuntimeException: Unable to
> > > read/process
> > > CSR:
> > > Command failed due to Exception: java.lang.Exception
> > >
> > > Message: Failed to authentication SSH user root on host 10.15.0.160
> > >
> > >
> > >
> > > at
> > > org.apache.cloudstack.ca
> > > .provider.RootCAProvider.generateCertificateUsingCsr
> > > (RootCAProvider.java:170)
> > >
> > > at
> > > org.apache.cloudstack.ca
> > > .provider.RootCAProvider.issueCertificate(RootCAProv
> > > ider.java:230)
> > >
> > > at
> > > org.apache.cloudstack.ca
> > > .CAManagerImpl.issueCertificate(CAManagerImpl.java:1
> > > 64)
> > >
> > > at
> > > jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
> > > Source)
> > >
> > > at
> > >
> > > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(D
> > > el
> > > egatin
> > > gMethodAccessorImpl.java:43)
> > >
> > > at
> > > java.base/java.lang.reflect.Method.invoke(Method.java:566)
> > >
> > > at
> > >
> > > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflect
> > > io
> > > n(AopU
> > > tils.java:344)
> > >
> > > at
> > >
> > > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJ
> > > oi
> > > npoint
> > > (ReflectiveMethodInvocation.java:198)
> > >
> > > at
> > >
> > > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > > (R
> > > eflect
> > > iveMethodInvocation.java:163)
> > >
> > > at
> > >
> > > org.apache.cloudstack.network.contrail.management.EventUtils$EventIn
> > > te
> > > rcepto
> > > r.invoke(EventUtils.java:107)
> > >
> > > at
> > >
> > > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > > (R
> > > eflect
> > > iveMethodInvocation.java:175)
> > >
> > > at
> > >
> > > com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor
> > > .j
> > > ava:51
> > > )
> > >
> > > at
> > >
> > > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > > (R
> > > eflect
> > > iveMethodInvocation.java:175)
> > >
> > > at
> > >
> > > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invo
> > > ke
> > > (Expos
> > > eInvocationInterceptor.java:97)
> > >
> > > at
> > >
> > > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > > (R
> > > eflect
> > > iveMethodInvocation.java:186)
> > >
> > > at
> > >
> > > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynam
> > > ic
> > > AopPro
> > > xy.java:215)
> > >
> > > at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
> > >
> > > at
> > >
> > > com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMac
> > > hi
> > > neMana
> > > gerImpl.java:970)
> > >
> > > at
> > >
> > > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachi
> > > ne
> > > Manage
> > > rImpl.java:1241)
> > >
> > > at
> > >
> > > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachi
> > > ne
> > > Manage
> > > rImpl.java:5502)
> > >
> > > at
> > > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Nati
> > > ve
> > > Method)
> > >
> > > at
> > >
> > > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Nativ
> > > eM
> > > ethodA
> > > ccessorImpl.java:62)
> > >
> > > at
> > >
> > > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(D
> > > el
> > > egatin
> > > gMethodAccessorImpl.java:43)
> > >
> > > at
> > > java.base/java.lang.reflect.Method.invoke(Method.java:566)
> > >
> > > at
> > >
> > > com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerP
> > > ro
> > > xy.jav
> > > a:107)
> > >
> > > at
> > >
> > > com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachin
> > > eM
> > > anager
> > > Impl.java:5669)
> > >
> > > at
> > > com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102
> > > )
> > >
> > > at
> > >
> > > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runI
> > > nC
> > > ontext
> > > (AsyncJobManagerImpl.java:620)
> > >
> > > at
> > >
> > > org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(M
> > > an
> > > agedCo
> > > ntextRunnable.java:48)
> > >
> > > at
> > >
> > > org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.c
> > > al
> > > l(Defa
> > > ultManagedContext.java:55)
> > >
> > > at
> > >
> > > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.cal
> > > lW
> > > ithCon
> > > text(DefaultManagedContext.java:102)
> > >
> > > at
> > >
> > > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.run
> > > Wi
> > > thCont
> > > ext(DefaultManagedContext.java:52)
> > >
> > > at
> > >
> > > org.apache.cloudstack.managed.context.ManagedContextRunnable.run(Man
> > > ag
> > > edCont
> > > extRunnable.java:45)
> > >
> > > at
> > >
> > > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(
> > > As
> > > yncJob
> > > ManagerImpl.java:568)
> > >
> > > at
> > >
> > > java.base/java.util.concurrent.Executors$RunnableAdapter.call(Execut
> > > or
> > > s.java
> > > :515)
> > >
> > > at
> > > java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> > >
> > > at
> > >
> > > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPo
> > > ol
> > > Execut
> > > or.java:1128)
> > >
> > > at
> > >
> > > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadP
> > > oo
> > > lExecu
> > > tor.java:628)
> > >
> > > at java.base/java.lang.Thread.run(Thread.java:829)
> > >
> > >
> > >
> > >
> > >
> > >
> > >
> > > Regards,
> > >
> > > Cristian
> > >
> > >
> >
> > --
> > Daan
> >
> >
>
> --
> Daan
>
>
--
Daan
RE: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Posted by cr...@istream.today.
Hi,
Yes, it is the server, but I already started to check the BIOS ... and I have very interesting question, why with KVM is working? What is OS different between KVM and VMware when comes to systemvm patching/inject keys? I think here I have the answer to may error.
Regards,
Cristian
-----Original Message-----
From: Daan Hoogland <da...@gmail.com>
Sent: Tuesday, September 28, 2021 12:14 PM
To: users <us...@cloudstack.apache.org>
Subject: Re: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
ok, than I would like to add an idea to your "it's the server". Not saying you are wrong but location points to network i.e. firewall(s).
Hope you strike luck,
On Tue, Sep 28, 2021 at 10:58 AM <cr...@istream.today> wrote:
> Hi,
>
> We already tested in all the ways is possible, this problem is
> related to these specific servers, 100%.
>
> The last test we did, was to add the ESXI server to the zone we
> already have and works, we disabled all hosts from that zone and
> destroyed the SSVM to force a redeploy on this ESXI, of course, we
> have ended with the same error.
>
> I want to mention that we have the exact same ESXI version and
> hardware, for me looks like there is a BIOS/HW/DISK issue.
>
>
> FYI:
>
> "I did this in 3 different new Setup (new Management install, NFS,
> etc), no matter if I add as first zone or second zone.
>
> " I have tested this with multiple Cloudstack versions, 4.15.0,
> 4.15.1,
> 4.15.2 and VMware, 6.5 and with 6.7 different patches and with the
> last patches. I'm 100% that is related to these 2 servers, I do not
> understand what is wrong with these servers, this is the problem.
>
> We have 2 identical servers' hardware/ESXI, the only difference is
> the location, the not working, are in UK, and any other location we
> have, DE, NYC, FR, CA, etc. works fine. But we tested with the exact
> same servers which are from DE(Germany) works perfect, with any ACS or
> VMware version (also same CIDR range/size for public and private),
> when we test with UK, we get same error no matter what."
>
> I want to mention that we have these servers (UK, DE, FR, CA ) from
> OVH in vRack and the UK, are not working. ( I manage these server
> for more than 5 years )"
>
> Regards,
> Cristian
>
> -----Original Message-----
> From: Daan Hoogland <da...@gmail.com>
> Sent: Tuesday, September 28, 2021 11:00 AM
> To: users <us...@cloudstack.apache.org>
> Subject: Re: Unable to read/process CSR: Command failed due to Exception:
> java.lang.Exception
>
> Christian, did you solve this (i just encountered your mail and see it
> is yet unanswered) the problem is with the certificate generation, it
> seems to try and generate from an empty string (the csr/certificate
> signing request) Not being able to sign in to the SVM is a problem but
> on vmware you would use a specific command as described in [1]. Did
> you try `ssh -i /opt/xensource/bin/id_rsa --p 3922 root@privateIP
> <ro...@10.209.161.128> OfTheHost`?
>
> [1]
>
> https://cwiki.apache.org/confluence/display/CLOUDSTACK/SSVM%2C+templat
> es%2C+Secondary+storage+troubleshooting
>
> On Thu, Sep 23, 2021 at 11:31 AM <cr...@istream.today> wrote:
>
> > Hello,
> >
> >
> >
> > I'm not sure why I get this error (added new vmware zone),
> > there are not permission issues for systemvms folder, and I do not
> > see any error above this. If I try to login to login into SSVM/Proxy
> > console via cloudstack management, I get this access denied.
> >
> >
> >
> > Any suggestion?
> >
> >
> >
> >
> >
> >
> >
> >
> >
> > 021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication
> > SSH user root on host 10.15.0.160
> >
> > 2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get
> > message for exception: Failed to authentication SSH user root on
> > host
> > 10.15.0.160
> >
> > 2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to
> > Exception:
> > java.lang.Exception
> >
> > Message: Failed to authentication SSH user root on host 10.15.0.160
> >
> >
> >
> > 2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution
> > result:
> > false
> >
> > 2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
> > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> 40-1394708509601300577:
> > Response Received:
> >
> > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> 40-1394708509601300577:
> > Processing: { Ans: , MgmtId: 345049356158, via:
> > 40(lnd-uk-001.shape.host),
> > Ver: v1, Flags: 10,
> > [{"org.apache.cloudstack.ca
> > .SetupKeystoreAnswer":{"result":"true","wait":"0"
> > ,"bypassHostMaintenance":"false"}}] }
> >
> > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > (logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: , MgmtId:
> > 345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
> > SetupKeystoreAnswer } }
> >
> > 2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
> > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > (logid:f3cd2252) Retrying after catching exception while trying to
> > secure agent for systemvm id=1955
> >
> > com.cloud.utils.exception.CloudRuntimeException: Unable to
> > read/process
> > CSR:
> > Command failed due to Exception: java.lang.Exception
> >
> > Message: Failed to authentication SSH user root on host 10.15.0.160
> >
> >
> >
> > at
> > org.apache.cloudstack.ca
> > .provider.RootCAProvider.generateCertificateUsingCsr
> > (RootCAProvider.java:170)
> >
> > at
> > org.apache.cloudstack.ca
> > .provider.RootCAProvider.issueCertificate(RootCAProv
> > ider.java:230)
> >
> > at
> > org.apache.cloudstack.ca
> > .CAManagerImpl.issueCertificate(CAManagerImpl.java:1
> > 64)
> >
> > at
> > jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
> > Source)
> >
> > at
> >
> > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(D
> > el
> > egatin
> > gMethodAccessorImpl.java:43)
> >
> > at
> > java.base/java.lang.reflect.Method.invoke(Method.java:566)
> >
> > at
> >
> > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflect
> > io
> > n(AopU
> > tils.java:344)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJ
> > oi
> > npoint
> > (ReflectiveMethodInvocation.java:198)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > (R
> > eflect
> > iveMethodInvocation.java:163)
> >
> > at
> >
> > org.apache.cloudstack.network.contrail.management.EventUtils$EventIn
> > te
> > rcepto
> > r.invoke(EventUtils.java:107)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > (R
> > eflect
> > iveMethodInvocation.java:175)
> >
> > at
> >
> > com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor
> > .j
> > ava:51
> > )
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > (R
> > eflect
> > iveMethodInvocation.java:175)
> >
> > at
> >
> > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invo
> > ke
> > (Expos
> > eInvocationInterceptor.java:97)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed
> > (R
> > eflect
> > iveMethodInvocation.java:186)
> >
> > at
> >
> > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynam
> > ic
> > AopPro
> > xy.java:215)
> >
> > at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMac
> > hi
> > neMana
> > gerImpl.java:970)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachi
> > ne
> > Manage
> > rImpl.java:1241)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachi
> > ne
> > Manage
> > rImpl.java:5502)
> >
> > at
> > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Nati
> > ve
> > Method)
> >
> > at
> >
> > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Nativ
> > eM
> > ethodA
> > ccessorImpl.java:62)
> >
> > at
> >
> > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(D
> > el
> > egatin
> > gMethodAccessorImpl.java:43)
> >
> > at
> > java.base/java.lang.reflect.Method.invoke(Method.java:566)
> >
> > at
> >
> > com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerP
> > ro
> > xy.jav
> > a:107)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachin
> > eM
> > anager
> > Impl.java:5669)
> >
> > at
> > com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102
> > )
> >
> > at
> >
> > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runI
> > nC
> > ontext
> > (AsyncJobManagerImpl.java:620)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(M
> > an
> > agedCo
> > ntextRunnable.java:48)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.c
> > al
> > l(Defa
> > ultManagedContext.java:55)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.cal
> > lW
> > ithCon
> > text(DefaultManagedContext.java:102)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.run
> > Wi
> > thCont
> > ext(DefaultManagedContext.java:52)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.ManagedContextRunnable.run(Man
> > ag
> > edCont
> > extRunnable.java:45)
> >
> > at
> >
> > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(
> > As
> > yncJob
> > ManagerImpl.java:568)
> >
> > at
> >
> > java.base/java.util.concurrent.Executors$RunnableAdapter.call(Execut
> > or
> > s.java
> > :515)
> >
> > at
> > java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> >
> > at
> >
> > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPo
> > ol
> > Execut
> > or.java:1128)
> >
> > at
> >
> > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadP
> > oo
> > lExecu
> > tor.java:628)
> >
> > at java.base/java.lang.Thread.run(Thread.java:829)
> >
> >
> >
> >
> >
> >
> >
> > Regards,
> >
> > Cristian
> >
> >
>
> --
> Daan
>
>
--
Daan
Re: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Posted by Daan Hoogland <da...@gmail.com>.
ok, than I would like to add an idea to your "it's the server". Not saying
you are wrong but location points to network i.e. firewall(s).
Hope you strike luck,
On Tue, Sep 28, 2021 at 10:58 AM <cr...@istream.today> wrote:
> Hi,
>
> We already tested in all the ways is possible, this problem is related
> to these specific servers, 100%.
>
> The last test we did, was to add the ESXI server to the zone we already
> have and works, we disabled all hosts from that zone and destroyed the SSVM
> to force a redeploy on this ESXI, of course, we have ended with the same
> error.
>
> I want to mention that we have the exact same ESXI version and
> hardware, for me looks like there is a BIOS/HW/DISK issue.
>
>
> FYI:
>
> "I did this in 3 different new Setup (new Management install, NFS,
> etc), no matter if I add as first zone or second zone.
>
> " I have tested this with multiple Cloudstack versions, 4.15.0, 4.15.1,
> 4.15.2 and VMware, 6.5 and with 6.7 different patches and with the last
> patches. I'm 100% that is related to these 2 servers, I do not understand
> what is wrong with these servers, this is the problem.
>
> We have 2 identical servers' hardware/ESXI, the only difference is the
> location, the not working, are in UK, and any other location we have, DE,
> NYC, FR, CA, etc. works fine. But we tested with the exact same servers
> which are from DE(Germany) works perfect, with any ACS or VMware version
> (also same CIDR range/size for public and private), when we test with UK,
> we get same error no matter what."
>
> I want to mention that we have these servers (UK, DE, FR, CA ) from OVH
> in vRack and the UK, are not working. ( I manage these server for more
> than 5 years )"
>
> Regards,
> Cristian
>
> -----Original Message-----
> From: Daan Hoogland <da...@gmail.com>
> Sent: Tuesday, September 28, 2021 11:00 AM
> To: users <us...@cloudstack.apache.org>
> Subject: Re: Unable to read/process CSR: Command failed due to Exception:
> java.lang.Exception
>
> Christian, did you solve this (i just encountered your mail and see it is
> yet unanswered) the problem is with the certificate generation, it seems to
> try and generate from an empty string (the csr/certificate signing request)
> Not being able to sign in to the SVM is a problem but on vmware you would
> use a specific command as described in [1]. Did you try `ssh -i
> /opt/xensource/bin/id_rsa --p 3922 root@privateIP <ro...@10.209.161.128>
> OfTheHost`?
>
> [1]
>
> https://cwiki.apache.org/confluence/display/CLOUDSTACK/SSVM%2C+templates%2C+Secondary+storage+troubleshooting
>
> On Thu, Sep 23, 2021 at 11:31 AM <cr...@istream.today> wrote:
>
> > Hello,
> >
> >
> >
> > I'm not sure why I get this error (added new vmware zone),
> > there are not permission issues for systemvms folder, and I do not see
> > any error above this. If I try to login to login into SSVM/Proxy
> > console via cloudstack management, I get this access denied.
> >
> >
> >
> > Any suggestion?
> >
> >
> >
> >
> >
> >
> >
> >
> >
> > 021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication
> > SSH user root on host 10.15.0.160
> >
> > 2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get
> > message for exception: Failed to authentication SSH user root on host
> > 10.15.0.160
> >
> > 2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to
> > Exception:
> > java.lang.Exception
> >
> > Message: Failed to authentication SSH user root on host 10.15.0.160
> >
> >
> >
> > 2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
> > (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> > job-34950/job-34972,
> > cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution
> > result:
> > false
> >
> > 2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
> > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> 40-1394708509601300577:
> > Response Received:
> >
> > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq
> 40-1394708509601300577:
> > Processing: { Ans: , MgmtId: 345049356158, via:
> > 40(lnd-uk-001.shape.host),
> > Ver: v1, Flags: 10,
> > [{"org.apache.cloudstack.ca
> > .SetupKeystoreAnswer":{"result":"true","wait":"0"
> > ,"bypassHostMaintenance":"false"}}] }
> >
> > 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > (logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: , MgmtId:
> > 345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
> > SetupKeystoreAnswer } }
> >
> > 2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
> > (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> > (logid:f3cd2252) Retrying after catching exception while trying to
> > secure agent for systemvm id=1955
> >
> > com.cloud.utils.exception.CloudRuntimeException: Unable to
> > read/process
> > CSR:
> > Command failed due to Exception: java.lang.Exception
> >
> > Message: Failed to authentication SSH user root on host 10.15.0.160
> >
> >
> >
> > at
> > org.apache.cloudstack.ca
> > .provider.RootCAProvider.generateCertificateUsingCsr
> > (RootCAProvider.java:170)
> >
> > at
> > org.apache.cloudstack.ca
> > .provider.RootCAProvider.issueCertificate(RootCAProv
> > ider.java:230)
> >
> > at
> > org.apache.cloudstack.ca
> > .CAManagerImpl.issueCertificate(CAManagerImpl.java:1
> > 64)
> >
> > at
> > jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
> > Source)
> >
> > at
> >
> > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Del
> > egatin
> > gMethodAccessorImpl.java:43)
> >
> > at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> >
> > at
> >
> > org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflectio
> > n(AopU
> > tils.java:344)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoi
> > npoint
> > (ReflectiveMethodInvocation.java:198)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> > eflect
> > iveMethodInvocation.java:163)
> >
> > at
> >
> > org.apache.cloudstack.network.contrail.management.EventUtils$EventInte
> > rcepto
> > r.invoke(EventUtils.java:107)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> > eflect
> > iveMethodInvocation.java:175)
> >
> > at
> >
> > com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.j
> > ava:51
> > )
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> > eflect
> > iveMethodInvocation.java:175)
> >
> > at
> >
> > org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke
> > (Expos
> > eInvocationInterceptor.java:97)
> >
> > at
> >
> > org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> > eflect
> > iveMethodInvocation.java:186)
> >
> > at
> >
> > org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamic
> > AopPro
> > xy.java:215)
> >
> > at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMachi
> > neMana
> > gerImpl.java:970)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachine
> > Manage
> > rImpl.java:1241)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachine
> > Manage
> > rImpl.java:5502)
> >
> > at
> > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> >
> > at
> >
> > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeM
> > ethodA
> > ccessorImpl.java:62)
> >
> > at
> >
> > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Del
> > egatin
> > gMethodAccessorImpl.java:43)
> >
> > at java.base/java.lang.reflect.Method.invoke(Method.java:566)
> >
> > at
> >
> > com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerPro
> > xy.jav
> > a:107)
> >
> > at
> >
> > com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineM
> > anager
> > Impl.java:5669)
> >
> > at
> > com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
> >
> > at
> >
> > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInC
> > ontext
> > (AsyncJobManagerImpl.java:620)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(Man
> > agedCo
> > ntextRunnable.java:48)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.cal
> > l(Defa
> > ultManagedContext.java:55)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callW
> > ithCon
> > text(DefaultManagedContext.java:102)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWi
> > thCont
> > ext(DefaultManagedContext.java:52)
> >
> > at
> >
> > org.apache.cloudstack.managed.context.ManagedContextRunnable.run(Manag
> > edCont
> > extRunnable.java:45)
> >
> > at
> >
> > org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(As
> > yncJob
> > ManagerImpl.java:568)
> >
> > at
> >
> > java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executor
> > s.java
> > :515)
> >
> > at
> > java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> >
> > at
> >
> > java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPool
> > Execut
> > or.java:1128)
> >
> > at
> >
> > java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoo
> > lExecu
> > tor.java:628)
> >
> > at java.base/java.lang.Thread.run(Thread.java:829)
> >
> >
> >
> >
> >
> >
> >
> > Regards,
> >
> > Cristian
> >
> >
>
> --
> Daan
>
>
--
Daan
RE: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Posted by cr...@istream.today.
Hi,
We already tested in all the ways is possible, this problem is related to these specific servers, 100%.
The last test we did, was to add the ESXI server to the zone we already have and works, we disabled all hosts from that zone and destroyed the SSVM to force a redeploy on this ESXI, of course, we have ended with the same error.
I want to mention that we have the exact same ESXI version and hardware, for me looks like there is a BIOS/HW/DISK issue.
FYI:
"I did this in 3 different new Setup (new Management install, NFS, etc), no matter if I add as first zone or second zone.
" I have tested this with multiple Cloudstack versions, 4.15.0, 4.15.1,
4.15.2 and VMware, 6.5 and with 6.7 different patches and with the last patches. I'm 100% that is related to these 2 servers, I do not understand what is wrong with these servers, this is the problem.
We have 2 identical servers' hardware/ESXI, the only difference is the location, the not working, are in UK, and any other location we have, DE, NYC, FR, CA, etc. works fine. But we tested with the exact same servers which are from DE(Germany) works perfect, with any ACS or VMware version (also same CIDR range/size for public and private), when we test with UK, we get same error no matter what."
I want to mention that we have these servers (UK, DE, FR, CA ) from OVH in vRack and the UK, are not working. ( I manage these server for more than 5 years )"
Regards,
Cristian
-----Original Message-----
From: Daan Hoogland <da...@gmail.com>
Sent: Tuesday, September 28, 2021 11:00 AM
To: users <us...@cloudstack.apache.org>
Subject: Re: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Christian, did you solve this (i just encountered your mail and see it is yet unanswered) the problem is with the certificate generation, it seems to try and generate from an empty string (the csr/certificate signing request) Not being able to sign in to the SVM is a problem but on vmware you would use a specific command as described in [1]. Did you try `ssh -i /opt/xensource/bin/id_rsa --p 3922 root@privateIP <ro...@10.209.161.128> OfTheHost`?
[1]
https://cwiki.apache.org/confluence/display/CLOUDSTACK/SSVM%2C+templates%2C+Secondary+storage+troubleshooting
On Thu, Sep 23, 2021 at 11:31 AM <cr...@istream.today> wrote:
> Hello,
>
>
>
> I'm not sure why I get this error (added new vmware zone),
> there are not permission issues for systemvms folder, and I do not see
> any error above this. If I try to login to login into SSVM/Proxy
> console via cloudstack management, I get this access denied.
>
>
>
> Any suggestion?
>
>
>
>
>
>
>
>
>
> 021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication
> SSH user root on host 10.15.0.160
>
> 2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get
> message for exception: Failed to authentication SSH user root on host
> 10.15.0.160
>
> 2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to
> Exception:
> java.lang.Exception
>
> Message: Failed to authentication SSH user root on host 10.15.0.160
>
>
>
> 2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host,
> job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution
> result:
> false
>
> 2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
> (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
> Response Received:
>
> 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
> Processing: { Ans: , MgmtId: 345049356158, via:
> 40(lnd-uk-001.shape.host),
> Ver: v1, Flags: 10,
> [{"org.apache.cloudstack.ca
> .SetupKeystoreAnswer":{"result":"true","wait":"0"
> ,"bypassHostMaintenance":"false"}}] }
>
> 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> (logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: , MgmtId:
> 345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
> SetupKeystoreAnswer } }
>
> 2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
> (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> (logid:f3cd2252) Retrying after catching exception while trying to
> secure agent for systemvm id=1955
>
> com.cloud.utils.exception.CloudRuntimeException: Unable to
> read/process
> CSR:
> Command failed due to Exception: java.lang.Exception
>
> Message: Failed to authentication SSH user root on host 10.15.0.160
>
>
>
> at
> org.apache.cloudstack.ca
> .provider.RootCAProvider.generateCertificateUsingCsr
> (RootCAProvider.java:170)
>
> at
> org.apache.cloudstack.ca
> .provider.RootCAProvider.issueCertificate(RootCAProv
> ider.java:230)
>
> at
> org.apache.cloudstack.ca
> .CAManagerImpl.issueCertificate(CAManagerImpl.java:1
> 64)
>
> at
> jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
> Source)
>
> at
>
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Del
> egatin
> gMethodAccessorImpl.java:43)
>
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>
> at
>
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflectio
> n(AopU
> tils.java:344)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoi
> npoint
> (ReflectiveMethodInvocation.java:198)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> eflect
> iveMethodInvocation.java:163)
>
> at
>
> org.apache.cloudstack.network.contrail.management.EventUtils$EventInte
> rcepto
> r.invoke(EventUtils.java:107)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> eflect
> iveMethodInvocation.java:175)
>
> at
>
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.j
> ava:51
> )
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> eflect
> iveMethodInvocation.java:175)
>
> at
>
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke
> (Expos
> eInvocationInterceptor.java:97)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(R
> eflect
> iveMethodInvocation.java:186)
>
> at
>
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamic
> AopPro
> xy.java:215)
>
> at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMachi
> neMana
> gerImpl.java:970)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachine
> Manage
> rImpl.java:1241)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachine
> Manage
> rImpl.java:5502)
>
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>
> at
>
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeM
> ethodA
> ccessorImpl.java:62)
>
> at
>
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Del
> egatin
> gMethodAccessorImpl.java:43)
>
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>
> at
>
> com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerPro
> xy.jav
> a:107)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineM
> anager
> Impl.java:5669)
>
> at
> com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
>
> at
>
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInC
> ontext
> (AsyncJobManagerImpl.java:620)
>
> at
>
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(Man
> agedCo
> ntextRunnable.java:48)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.cal
> l(Defa
> ultManagedContext.java:55)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callW
> ithCon
> text(DefaultManagedContext.java:102)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWi
> thCont
> ext(DefaultManagedContext.java:52)
>
> at
>
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(Manag
> edCont
> extRunnable.java:45)
>
> at
>
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(As
> yncJob
> ManagerImpl.java:568)
>
> at
>
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executor
> s.java
> :515)
>
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>
> at
>
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPool
> Execut
> or.java:1128)
>
> at
>
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoo
> lExecu
> tor.java:628)
>
> at java.base/java.lang.Thread.run(Thread.java:829)
>
>
>
>
>
>
>
> Regards,
>
> Cristian
>
>
--
Daan
Re: Unable to read/process CSR: Command failed due to Exception: java.lang.Exception
Posted by Daan Hoogland <da...@gmail.com>.
Christian, did you solve this (i just encountered your mail and see it is
yet unanswered)
the problem is with the certificate generation, it seems to try and
generate from an empty string (the csr/certificate signing request)
Not being able to sign in to the SVM is a problem but on vmware you would
use a specific command as described in [1]. Did you try `ssh -i
/opt/xensource/bin/id_rsa --p 3922 root@privateIP <ro...@10.209.161.128>
OfTheHost`?
[1]
https://cwiki.apache.org/confluence/display/CLOUDSTACK/SSVM%2C+templates%2C+Secondary+storage+troubleshooting
On Thu, Sep 23, 2021 at 11:31 AM <cr...@istream.today> wrote:
> Hello,
>
>
>
> I'm not sure why I get this error (added new vmware zone), there are
> not permission issues for systemvms folder, and I do not see any error
> above
> this. If I try to login to login into SSVM/Proxy console via cloudstack
> management, I get this access denied.
>
>
>
> Any suggestion?
>
>
>
>
>
>
>
>
>
> 021-09-23 05:22:15,012 ERROR [c.c.u.s.SshHelper]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) Failed to authentication SSH
> user root on host 10.15.0.160
>
> 2021-09-23 05:22:15,012 INFO [c.c.h.v.u.VmwareHelper]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) [ignored]failed to get message
> for exception: Failed to authentication SSH user root on host 10.15.0.160
>
> 2021-09-23 05:22:15,012 ERROR [c.c.h.v.r.VmwareResource]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) Command failed due to
> Exception:
> java.lang.Exception
>
> Message: Failed to authentication SSH user root on host 10.15.0.160
>
>
>
> 2021-09-23 05:22:15,012 DEBUG [c.c.h.v.r.VmwareResource]
> (DirectAgent-283:ctx-b1e493bd lnd-uk-001.shape.host, job-34950/job-34972,
> cmd: SetupKeyStoreCommand) (logid:f3cd2252) keystore-setup execution
> result:
> false
>
> 2021-09-23 05:22:15,014 DEBUG [c.c.a.m.DirectAgentAttache]
> (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
> Response Received:
>
> 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> (DirectAgent-283:ctx-b1e493bd) (logid:f3cd2252) Seq 40-1394708509601300577:
> Processing: { Ans: , MgmtId: 345049356158, via: 40(lnd-uk-001.shape.host),
> Ver: v1, Flags: 10,
> [{"org.apache.cloudstack.ca
> .SetupKeystoreAnswer":{"result":"true","wait":"0"
> ,"bypassHostMaintenance":"false"}}] }
>
> 2021-09-23 05:22:15,016 DEBUG [c.c.a.t.Request]
> (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> (logid:f3cd2252) Seq 40-1394708509601300577: Received: { Ans: , MgmtId:
> 345049356158, via: 40(lnd-uk-001.test.host), Ver: v1, Flags: 10, {
> SetupKeystoreAnswer } }
>
> 2021-09-23 05:22:15,022 ERROR [c.c.v.VirtualMachineManagerImpl]
> (Work-Job-Executor-14:ctx-10c822ed job-34950/job-34972 ctx-67aa0df0)
> (logid:f3cd2252) Retrying after catching exception while trying to secure
> agent for systemvm id=1955
>
> com.cloud.utils.exception.CloudRuntimeException: Unable to read/process
> CSR:
> Command failed due to Exception: java.lang.Exception
>
> Message: Failed to authentication SSH user root on host 10.15.0.160
>
>
>
> at
> org.apache.cloudstack.ca
> .provider.RootCAProvider.generateCertificateUsingCsr
> (RootCAProvider.java:170)
>
> at
> org.apache.cloudstack.ca
> .provider.RootCAProvider.issueCertificate(RootCAProv
> ider.java:230)
>
> at
> org.apache.cloudstack.ca
> .CAManagerImpl.issueCertificate(CAManagerImpl.java:1
> 64)
>
> at jdk.internal.reflect.GeneratedMethodAccessor318.invoke(Unknown
> Source)
>
> at
>
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Delegatin
> gMethodAccessorImpl.java:43)
>
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>
> at
>
> org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopU
> tils.java:344)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint
> (ReflectiveMethodInvocation.java:198)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
> iveMethodInvocation.java:163)
>
> at
>
> org.apache.cloudstack.network.contrail.management.EventUtils$EventIntercepto
> r.invoke(EventUtils.java:107)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
> iveMethodInvocation.java:175)
>
> at
>
> com.cloud.event.ActionEventInterceptor.invoke(ActionEventInterceptor.java:51
> )
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
> iveMethodInvocation.java:175)
>
> at
>
> org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(Expos
> eInvocationInterceptor.java:97)
>
> at
>
> org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(Reflect
> iveMethodInvocation.java:186)
>
> at
>
> org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopPro
> xy.java:215)
>
> at com.sun.proxy.$Proxy292.issueCertificate(Unknown Source)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.setupAgentSecurity(VirtualMachineMana
> gerImpl.java:970)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManage
> rImpl.java:1241)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManage
> rImpl.java:5502)
>
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>
> at
>
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodA
> ccessorImpl.java:62)
>
> at
>
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Delegatin
> gMethodAccessorImpl.java:43)
>
> at java.base/java.lang.reflect.Method.invoke(Method.java:566)
>
> at
>
> com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.jav
> a:107)
>
> at
>
> com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManager
> Impl.java:5669)
>
> at
> com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
>
> at
>
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext
> (AsyncJobManagerImpl.java:620)
>
> at
>
> org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedCo
> ntextRunnable.java:48)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(Defa
> ultManagedContext.java:55)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithCon
> text(DefaultManagedContext.java:102)
>
> at
>
> org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithCont
> ext(DefaultManagedContext.java:52)
>
> at
>
> org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedCont
> extRunnable.java:45)
>
> at
>
> org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJob
> ManagerImpl.java:568)
>
> at
>
> java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java
> :515)
>
> at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>
> at
>
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecut
> or.java:1128)
>
> at
>
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecu
> tor.java:628)
>
> at java.base/java.lang.Thread.run(Thread.java:829)
>
>
>
>
>
>
>
> Regards,
>
> Cristian
>
>
--
Daan