SDDC Manager UI is stuck on the "VMware Cloud Foundation is initializing" screen

Products

• VMware SDDC Manager

Issue/Introduction

•   SDDC Manager UI fails to load 

less var/log/vmware/vcf/sddc-manager-ui-app/sddcManagerServer.log

YYYY-MM-DD ERROR [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:50] axios.error.code  {"code":"ERR_BAD_RESPONSE"}
YYYY-MM-DD ERROR [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:51] axios.error.config {"config":{"adapter":["xhr","http"],"env":{},"headers":{"Accept":"applica
tion/json, text/plain, */*","Accept-Encoding":"gzip, compress, deflate, br","User-Agent":"axios/1.6.2","X-B3-SpanId":"46e3ccf847804baa","X-B3-TraceId":"faf873e40ed6472d"},"maxBodyLength":-1,"maxContentLength":-1,"method":"get","timeout":0,"transformRequest":[null]
,"transformResponse":[null],"transitional":{"clarifyTimeoutError":false,"forcedJSONParsing":true,"silentJSONParsing":true},"url":"http://127.0.0.1/commonsvcs/idpMetadata","xsrfCookieName":"XSRF-TOKEN","xsrfHeaderName":"X-XSRF-TOKEN"}}
YYYY-MM-DDERROR  [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:56] axios.error.response.status {"status":500}
YYYY-MM-DD ERROR  [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:57] axios.error.response.headers {"headers":{"cache-control":"no-cache, no-store, max-age=0, mus
t-revalidate","connection":"close","content-type":"application/json","date":"Fri, DD MM YYYY TT: GMT","expires":"0","pragma":"no-cache","server":"nginx","x-content-type-options":"nosniff","x-frame-options":"DENY","x-xss-protection":"0"}}
YYYY-MM-DD ERROR [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:58] axios.error.response.data {"data":"cipher arguments do not contain a required key: iv"}
YYYY-MM-DD ERROR [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:69] axios.error.stack {"stack":"AxiosError: Request failed with status code 500\n    at settle (
/opt/vmware/vcf/sddc-manager-ui-app/server/node_modules/axios/dist/node/axios.cjs:1967:12)\n    at IncomingMessage.handleStreamEnd (/opt/vmware/vcf/sddc-manager-ui-app/server/node_modules/axios/dist/node/axios.cjs:3062:11)\n    at IncomingMessage.emit (node:events
:536:35)\n    at endReadableNT (node:internal/streams/readable:1698:12)\n    at process.processTicksAndRejections (node:internal/process/task_queues:90:21)"}
YYYY-MM-DD ERROR  [services/wrappers/requestPromiseWrapper.js, init-pscs-primary, logAxiosError:74] --- Axios Error End ---
YYYY-MM-DD VERBOSE [services/logging/opentrace.js, init-pscs-primary, _finish:231] Span init-pscs-primary complete: Total time: 2087ms {"duration":2087}
YYYY-MM-DD ERROR  [services/pscUtils.js, init-pscs, attemptPSCInit:67] Caught error from await primaryPscInit
YYYY-MM-DD WARN  [services/pscUtils.js, init-pscs, attemptPSCInitWithRetry:111]
100.109: VError: PSC Initialization attempt "23" failed: Failed to initiate PSC: Primary psc init failed and failover psc init also failed: Unable to retrieve iDP Metadata: Request failed with status code 500
    at Object.initializationPscError (/opt/vmware/vcf/sddc-manager-ui-app/server/src/errors/VCFError.js:104:5)
    at attemptPSCInitWithRetry (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/pscUtils.js:104:46)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
Error Info: {"retryCount":23,"status":403,"errorModule":100,"errorCode":109,"code":"ERR_BAD_RESPONSE"}
caused by:
100.108: VError: Failed to initiate PSC: Primary psc init failed and failover psc init also failed: Unable to retrieve iDP Metadata: Request failed with status code 500
    at Object.initiatePscError (/opt/vmware/vcf/sddc-manager-ui-app/server/src/errors/VCFError.js:104:5)
    at attemptPSCInit (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/pscUtils.js:72:26)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
    at async attemptPSCInitWithRetry (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/pscUtils.js:99:17)
Error Info: {"errorModule":100,"errorCode":108,"code":"ERR_BAD_RESPONSE"}
caused by:
100.107: VError: Primary psc init failed and failover psc init also failed: Unable to retrieve iDP Metadata: Request failed with status code 500
    at Object.primaryPscInitError (/opt/vmware/vcf/sddc-manager-ui-app/server/src/errors/VCFError.js:104:5)
    at attemptPSCInit (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/pscUtils.js:68:30)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
    at async attemptPSCInitWithRetry (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/pscUtils.js:99:17)
Error Info: {"errorModule":100,"errorCode":107,"code":"ERR_BAD_RESPONSE"}
caused by:
100.149: VError: Unable to retrieve iDP Metadata: Request failed with status code 500
    at Object.unableToRetrieveIDPMetadataError (/opt/vmware/vcf/sddc-manager-ui-app/server/src/errors/VCFError.js:104:5)
    at retrieveIDPMetadataRest (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/sso-initialization.js:112:30)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
    at async initializeSSO (/opt/vmware/vcf/sddc-manager-ui-app/server/src/services/sso-initialization.js:55:27)
Error Info: {"errorModule":100,"errorCode":149,"code":"ERR_BAD_RESPONSE"}
caused by:
AxiosError: Request failed with status code 500
All Errors Info:
 {"code":"ERR_BAD_RESPONSE","retryCount":23,"status":403}

Environment

SDDC manager 9.x

Cause

This issue occurs when credential table in the SDDC Manager database has inconsistency 

Resolution

Resolution requires clearing/alerting database tables . Please open a ticket with Broadcom support for assistance.