Edit

Secrets Vault server sealed after server reboot or restart

WorkFusion components that get values from the Secrets Vault server are restarting indefinitely.

workfusion.out.log example

[localhost-startStop-1] ContextLoader [ERROR] Context initialization failed
org.springframework.beans.factory.UnsatisfiedDependencyException: Error creating bean with name 'entityManagerFactory' defined in class path resource [com/workfusion/controltower/core/repository/config/PersistenceConfig.class]: Unsatisfied dependency expressed through method 'entityManagerFactory' parameter 0; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'hazelcastInstance' defined in class path resource [com/workfusion/service/config/HazelcastConfig.class]: Bean instantiation via factory method failed; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [com.hazelcast.core.HazelcastInstance]: Factory method 'hzInstance' threw exception; nested exception is java.lang.IllegalArgumentException: argument 'name' can't be null
        at org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:749)
        at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:467)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1173)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1067)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:513)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483)
        at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306)
        at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)
        at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302)
        at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197)
        at org.springframework.context.support.AbstractApplicationContext.getBean(AbstractApplicationContext.java:1080)
        at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:857)
        at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543)
        at org.springframework.web.context.ContextLoader.configureAndRefreshWebApplicationContext(ContextLoader.java:443)
        at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:325)
        at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:107)
        at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:4792)
        at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5256)
        at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:150)
        at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:754)
        at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:730)
        at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:734)
        at org.apache.catalina.startup.HostConfig.deployDescriptor(HostConfig.java:629)
        at org.apache.catalina.startup.HostConfig$DeployDescriptor.run(HostConfig.java:1839)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'hazelcastInstance' defined in class path resource [com/workfusion/service/config/HazelcastConfig.class]: Bean instantiation via factory method failed; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [com.hazelcast.core.HazelcastInstance]: Factory method 'hzInstance' threw exception; nested exception is java.lang.IllegalArgumentException: argument 'name' can't be null
        at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:599)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1173)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1067)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:513)
        at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483)
        at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306)
        at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)
        at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302)
        at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197)
        at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:296)
        at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202)
        at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:208)
        at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1138)
        at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1066)
        at org.springframework.beans.factory.support.ConstructorResolver.resolveAutowiredArgument(ConstructorResolver.java:835)
        at org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:741)
        ... 28 more
Caused by: org.springframework.beans.BeanInstantiationException: Failed to instantiate [com.hazelcast.core.HazelcastInstance]: Factory method 'hzInstance' threw exception; nested exception is java.lang.IllegalArgumentException: argument 'name' can't be null
        at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:189)
        at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:588)
        ... 43 more
Caused by: java.lang.IllegalArgumentException: argument 'name' can't be null
        at com.hazelcast.util.Preconditions.isNotNull(Preconditions.java:88)
        at com.hazelcast.config.GroupConfig.setName(GroupConfig.java:86)
        at com.hazelcast.config.GroupConfig.<init>(GroupConfig.java:65)
        at com.workfusion.service.config.HazelcastConfig.hzInstance(HazelcastConfig.java:58)
        at com.workfusion.service.config.HazelcastConfig$$EnhancerBySpringCGLIB$$3edf6067.CGLIB$hzInstance$1(<generated>)
        at com.workfusion.service.config.HazelcastConfig$$EnhancerBySpringCGLIB$$3edf6067$$FastClassBySpringCGLIB$$dd228486.invoke(<generated>)
        at org.springframework.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:228)
        at org.springframework.context.annotation.ConfigurationClassEnhancer$BeanMethodInterceptor.intercept(ConfigurationClassEnhancer.java:358)
        at com.workfusion.service.config.HazelcastConfig$$EnhancerBySpringCGLIB$$3edf6067.hzInstance(<generated>)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:162)
        ... 44 more

Statement

The Secrets Vault data is encrypted and cannot be accessed by other components before it is decrypted.

Solution

Verify the Secrets Vault server status

To verify the Secrets Vault server status, follow these steps:

1. Connect to the DB server using SSH.

2. Go to the Secrets Vault folder:

   cd <install_dir>/vault
   

3. Check the Secrets Vault status:

   ./vault status -tls-skip-verify
   

After that, you have two options: Secrets Vault Server is not running or is sealed. Follow the steps for each case below.

Secrets Vault Server is not running

The following response means that the Secret Vault is not running:

Error checking seal status: Get https://127.0.0.1:8200/v1/sys/seal-status: dial tcp 127.0.0.1:8200: getsockopt: connection refused

1. You should start the Secret Vault server using the command below:

   Start Vault-server and unseal it

   wfmanager start vault:vault-server
   

2. Unseal the Secrets Vault by using the command below:

   wfmanager start vault:vault-unseal
   

Secret Vault sealed

1. Unseal the Secrets Vault by using the command below:

   wfmanager start vault:vault-unseal
   

2. Check logs:

   tail <install_dir>/supervisord/log/vault-unseal.out.log
   

   The following message in the wf.log means the unsealing process has been completed successfully:

   {"key":"5ab3b70bedf07f549e8e8dc2891c95f887aa773d6872109beaba0fc15e8850e301"}
   {"sealed":true,"t":5,"n":5,"progress":1,"nonce":"78b335ce-a266-f825-ff83-a7bf190f851f","version":"0.7.0"}
   {"key":"009804ad2748a332b1d2e7dfc810a3e367ed423d2a19a06e906f84fc20c2b9da02"}
   {"sealed":true,"t":5,"n":5,"progress":2,"nonce":"78b335ce-a266-f825-ff83-a7bf190f851f","version":"0.7.0"}
   {"key":"eaddc8ae95ead5943ca471205413b04640345eb948514945919e00c58482a62903"}
   {"sealed":true,"t":5,"n":5,"progress":3,"nonce":"78b335ce-a266-f825-ff83-a7bf190f851f","version":"0.7.0"}
   {"key":"2b2c2276293b5160265b5da3c53297508b1968ef0dd2da15f8bd3aa5c226976a04"}
   {"sealed":true,"t":5,"n":5,"progress":4,"nonce":"78b335ce-a266-f825-ff83-a7bf190f851f","version":"0.7.0"}
   {"key":"81fdd5494e2b356783fd761553a87da07fec28e013be1bbac1d41b281feca77005"}
   {"sealed":false,"t":5,"n":5,"progress":0,"nonce":"","version":"0.7.0","cluster_name":"vault-cluster-cf5f5c24","cluster_id":"c5dd2a3f-c057-f236-dace-33c4b19209d1"}
   

Unsealing failed

The following log output means the unsealing process was unsuccessful:

expiration state restore failed: failed to read lease entry: EOF

In this case, perform the following steps:

1. Stop the Secret Vault server:

   wfmanager stop vault:vault-server
   

2. Create the Secrets Vault backup copy:

   cp -pr <install_dir>/vault <install_dir>/vault-bk
   

3. Remove old sessions:

   rm -f <install_dir>/vault/data/sys/expire/id/auth/cert/login/*
   rm -f <install_dir>/vault/data/sys/token/id/*
   rm -f <install_dir>/vault/data/sys/token/accessor/*
   

4. Run the following commands in case you receive the List too long error:

   find <install_dir>/vault/data/sys/expire/id/auth/cert/login/ -type f -exec rm -f {} \;
   find <install_dir>/vault/data/sys/token/id/ -type f -exec rm -f {} \;
   find <install_dir>/vault/data/sys/token/accessor/ -type f -exec rm -f {} \;
   

5. Start the Secrets Vault server:

   wfmanager start vault:vault-server
   

6. Unseal the Secrets Vault server:

   wfmanager start vault:vault-unseal