diff --git a/apps/server-nestjs/package.json b/apps/server-nestjs/package.json index 99df988602..784c03a1ac 100644 --- a/apps/server-nestjs/package.json +++ b/apps/server-nestjs/package.json @@ -29,16 +29,9 @@ "test:debug": "vitest --inspect" }, "dependencies": { - "@cpn-console/argocd-plugin": "workspace:^", - "@cpn-console/gitlab-plugin": "workspace:^", - "@cpn-console/harbor-plugin": "workspace:^", "@cpn-console/hooks": "workspace:^", - "@cpn-console/keycloak-plugin": "workspace:^", "@cpn-console/logger": "workspace:^", - "@cpn-console/nexus-plugin": "workspace:^", "@cpn-console/shared": "workspace:^", - "@cpn-console/sonarqube-plugin": "workspace:^", - "@cpn-console/vault-plugin": "workspace:^", "@gitbeaker/core": "catalog:runtime", "@gitbeaker/requester-utils": "catalog:runtime", "@gitbeaker/rest": "catalog:runtime", diff --git a/apps/server-nestjs/src/main.module.ts b/apps/server-nestjs/src/main.module.ts index 0fdc0dce02..ffe46e2440 100644 --- a/apps/server-nestjs/src/main.module.ts +++ b/apps/server-nestjs/src/main.module.ts @@ -21,6 +21,7 @@ import { ProjectSecretsModule } from './modules/project-secrets/project-secrets. import { ProjectServicesModule } from './modules/project-services/project-services.module' import { ProjectModule } from './modules/project/project.module' import { RepositoryModule } from './modules/repository/repository.module' +import { ServiceMonitorModule } from './modules/service-monitor/service-monitor.module' import { StageModule } from './modules/stage/stage.module' import { SystemConfigModule } from './modules/system-config/system-config.module' import { SystemSettingsModule } from './modules/system-settings/system-settings.module' @@ -55,6 +56,7 @@ import { getDotenvPaths } from './utils/dotenv.utils' ProjectSecretsModule, ProjectServicesModule, RepositoryModule, + ServiceMonitorModule, ScheduleModule.forRoot(), StageModule, SystemConfigModule, diff --git a/apps/server-nestjs/src/modules/registry/registry-health.service.spec.ts b/apps/server-nestjs/src/modules/registry/registry-health.service.spec.ts new file mode 100644 index 0000000000..e221dfe1a4 --- /dev/null +++ b/apps/server-nestjs/src/modules/registry/registry-health.service.spec.ts @@ -0,0 +1,79 @@ +import type { HarborConfig } from '../../config/harbor.config' +import { HealthIndicatorService } from '@nestjs/terminus' +import { Test } from '@nestjs/testing' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { mockDeep } from 'vitest-mock-extended' +import { harborConfigFactory } from '../../config/harbor.config' +import { RegistryHealthService } from './registry-health.service' + +const harborUrl = 'https://harbor.example' +const healthUrl = `${harborUrl}/api/v2.0/health` + +const server = setupServer() + +describe('registryHealthService', () => { + let service: RegistryHealthService + + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) + + beforeEach(async () => { + const config = mockDeep({ + url: harborUrl, + internalUrl: undefined, + admin: 'admin', + adminPassword: 'password', + }) + + const module = await Test.createTestingModule({ + providers: [ + RegistryHealthService, + { provide: harborConfigFactory.KEY, useValue: config }, + HealthIndicatorService, + ], + }).compile() + + service = module.get(RegistryHealthService) + }) + afterEach(() => server.resetHandlers()) + afterAll(() => server.close()) + + it('reports up when every component is healthy', async () => { + server.use(http.get(healthUrl, () => HttpResponse.json({ status: 'healthy', components: [] }))) + + await expect(service.check()).resolves.toEqual({ harbor: { status: 'up' } }) + }) + + it('reports down when a core component is unhealthy', async () => { + server.use(http.get(healthUrl, () => HttpResponse.json({ + status: 'unhealthy', + components: [{ name: 'registryctl', status: 'unhealthy' }], + }))) + + await expect(service.check()).resolves.toEqual({ harbor: { status: 'down', message: 'Service en erreur' } }) + }) + + it('reports a degraded up when only a non-core component is unhealthy', async () => { + server.use(http.get(healthUrl, () => HttpResponse.json({ + status: 'unhealthy', + components: [{ name: 'jobservice', status: 'unhealthy' }], + }))) + + await expect(service.check()).resolves.toEqual({ harbor: { status: 'up', message: 'Service dégradé' } }) + }) + + it('reports down when the health endpoint does not answer with a 200', async () => { + server.use(http.get(healthUrl, () => new HttpResponse(null, { status: 503 }))) + + await expect(service.check()).resolves.toEqual({ harbor: { status: 'down', message: 'Fatal Error' } }) + }) + + it('reports down with the cause when the request fails', async () => { + server.use(http.get(healthUrl, () => HttpResponse.error())) + + await expect(service.check()).resolves.toMatchObject({ + harbor: { status: 'down', message: 'Erreur lors la requête', cause: expect.any(Error) }, + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/registry/registry-health.service.ts b/apps/server-nestjs/src/modules/registry/registry-health.service.ts index ef29547a6f..72a0d87d25 100644 --- a/apps/server-nestjs/src/modules/registry/registry-health.service.ts +++ b/apps/server-nestjs/src/modules/registry/registry-health.service.ts @@ -1,9 +1,21 @@ import type { HarborConfig } from '../../config/harbor.config' import { HttpStatus, Inject, Injectable } from '@nestjs/common' import { HealthIndicatorService } from '@nestjs/terminus' +import { z } from 'zod' import { harborConfigFactory } from '../../config/harbor.config' import { PLUGIN_NAME } from './registry.constants' +// /api/v2.0/health splits degraded from failed, unlike /api/v2.0/ping. +const CORE_COMPONENTS = new Set(['core', 'database', 'portal', 'registry', 'registryctl']) + +const harborHealthBodySchema = z.object({ + status: z.string(), + components: z.array(z.object({ + name: z.string(), + status: z.string(), + })).optional(), +}) + @Injectable() export class RegistryHealthService { constructor( @@ -13,17 +25,29 @@ export class RegistryHealthService { async check() { const indicator = this.healthIndicator.check(PLUGIN_NAME) - const headers: Record = {} - const credentials = `${this.harborConfig.admin}:${this.harborConfig.adminPassword}` - const base64 = Buffer.from(credentials).toString('base64') - headers.Authorization = `Basic ${base64}` try { - const url = new URL('/api/v2.0/ping', this.harborConfig.internalUrl ?? this.harborConfig.url).toString() - const response = await fetch(url, { method: 'GET', headers }) - if (response.status < HttpStatus.INTERNAL_SERVER_ERROR) return indicator.up({ httpStatus: response.status }) - return indicator.down({ httpStatus: response.status }) + const url = new URL('/api/v2.0/health', this.harborConfig.internalUrl ?? this.harborConfig.url).toString() + const response = await fetch(url, { method: 'GET', headers: this.authHeaders() }) + if (response.status !== HttpStatus.OK) return indicator.down({ message: 'Fatal Error' }) + + const parsed = harborHealthBodySchema.safeParse(await response.json()) + if (!parsed.success) return indicator.down({ message: 'Erreur lors la requête' }) + if (parsed.data.status === 'healthy') return indicator.up() + + const failedCoreComponent = parsed.data.components?.some(component => + component.status === 'unhealthy' + && CORE_COMPONENTS.has(component.name), + ) + return failedCoreComponent + ? indicator.down({ message: 'Service en erreur' }) + : indicator.up({ message: 'Service dégradé' }) } catch (error) { - return indicator.down(error instanceof Error ? error.message : String(error)) + return indicator.down({ message: 'Erreur lors la requête', cause: error }) } } + + private authHeaders(): Record { + const credentials = `${this.harborConfig.admin}:${this.harborConfig.adminPassword}` + return { Authorization: `Basic ${Buffer.from(credentials).toString('base64')}` } + } } diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.constants.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.constants.ts new file mode 100644 index 0000000000..371e97e7b8 --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.constants.ts @@ -0,0 +1,7 @@ +export const INTERVAL_MS = 5 * 60 * 1000 + +export const ERROR_MESSAGE = 'Service en erreur' + +export const REQUEST_ERROR_MESSAGE = 'Erreur lors la requête' + +export const PENDING_MESSAGE = 'En attente d\'une première vérification' diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.spec.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.spec.ts new file mode 100644 index 0000000000..a20b584e3f --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.spec.ts @@ -0,0 +1,60 @@ +import type { TestingModule } from '@nestjs/testing' +import type { MockProxy } from 'vitest-mock-extended' +import type { ServiceHealth } from './service-monitor.service' +import { MonitorStatus } from '@cpn-console/shared' +import { Test } from '@nestjs/testing' +import { beforeEach, describe, expect, it } from 'vitest' +import { mock } from 'vitest-mock-extended' +import { UserGuard } from '../infrastructure/permission/user/user.guard' +import { ServiceMonitorController } from './service-monitor.controller' +import { ServiceMonitorService } from './service-monitor.service' + +describe('serviceMonitorController', () => { + let module: TestingModule + let controller: ServiceMonitorController + let service: MockProxy + + beforeEach(async () => { + service = mock() + + module = await Test.createTestingModule({ + controllers: [ServiceMonitorController], + providers: [ + { provide: ServiceMonitorService, useValue: service }, + ], + }) + .overrideGuard(UserGuard) + .useValue({ canActivate: () => true }) + .compile() + + controller = module.get(ServiceMonitorController) + }) + + it('should be defined', () => { + expect(controller).toBeDefined() + }) + + it('strips the cause from the public route', async () => { + const health: ServiceHealth[] = [{ name: 'keycloak', status: MonitorStatus.ERROR, message: 'down', interval: 1000, lastUpdateTimestamp: 0, cause: 'connection refused' }] + service.getServiceHealth.mockResolvedValue(health) + + await expect(controller.getServiceHealth()).resolves.toEqual([{ name: 'keycloak', status: MonitorStatus.ERROR, message: 'down', interval: 1000, lastUpdateTimestamp: 0 }]) + expect(service.getServiceHealth).toHaveBeenCalledTimes(1) + }) + + it('delegates complete-services to the service, formatting the cause', async () => { + const health: ServiceHealth[] = [{ name: 'keycloak', status: MonitorStatus.OK, message: MonitorStatus.OK, interval: 1000, lastUpdateTimestamp: 0, cause: new Error('boom') }] + service.getServiceHealth.mockResolvedValue(health) + + await expect(controller.getCompleteServiceHealth()).resolves.toEqual([{ name: 'keycloak', status: MonitorStatus.OK, message: MonitorStatus.OK, interval: 1000, lastUpdateTimestamp: 0, cause: 'boom' }]) + expect(service.getServiceHealth).toHaveBeenCalledTimes(1) + }) + + it('delegates refresh-services to the service, formatting the cause', async () => { + const health: ServiceHealth[] = [{ name: 'keycloak', status: MonitorStatus.ERROR, message: 'down', interval: 1000, lastUpdateTimestamp: 0, cause: new Error('boom') }] + service.refreshServiceHealth.mockResolvedValue(health) + + await expect(controller.refreshServiceHealth()).resolves.toEqual([{ name: 'keycloak', status: MonitorStatus.ERROR, message: 'down', interval: 1000, lastUpdateTimestamp: 0, cause: 'boom' }]) + expect(service.refreshServiceHealth).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.ts new file mode 100644 index 0000000000..eb9c9c658e --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.controller.ts @@ -0,0 +1,41 @@ +import type { ServiceHealth } from './service-monitor.service' +import { Controller, Get, Inject, UseGuards } from '@nestjs/common' +import { RequireAdminPermission } from '../infrastructure/permission/user/user-admin-permission.decorator' +import { UserGuard } from '../infrastructure/permission/user/user.guard' +import { ServiceMonitorService } from './service-monitor.service' +import { toCause } from './service-monitor.utils' + +function toServiceHealth({ cause, ...health }: ServiceHealth): ServiceHealth { + return { ...health, ...(cause !== undefined ? { cause: toCause(cause) } : {}) } +} + +// MonitorInfos.cause is admin-only +function toPublicServiceHealth({ cause, ...health }: ServiceHealth): ServiceHealth { + return health +} + +@Controller('api/v1') +export class ServiceMonitorController { + constructor( + @Inject(ServiceMonitorService) private readonly service: ServiceMonitorService, + ) {} + + @Get('health-services') + getServiceHealth() { + return this.service.getServiceHealth().then(health => health.map(toPublicServiceHealth)) + } + + @Get('complete-services') + @UseGuards(UserGuard) + @RequireAdminPermission('ListSystem') + getCompleteServiceHealth() { + return this.service.getServiceHealth().then(health => health.map(toServiceHealth)) + } + + @Get('refresh-services') + @UseGuards(UserGuard) + @RequireAdminPermission('ManageSystem') + refreshServiceHealth() { + return this.service.refreshServiceHealth().then(health => health.map(toServiceHealth)) + } +} diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.module.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.module.ts new file mode 100644 index 0000000000..dbd3d5f851 --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.module.ts @@ -0,0 +1,32 @@ +import { CacheModule } from '@nestjs/cache-manager' +import { Module } from '@nestjs/common' +import { ConditionalModule } from '@nestjs/config' +import { ArgoCDModule } from '../argocd/argocd.module' +import { GitlabModule } from '../gitlab/gitlab.module' +import { AuthModule } from '../infrastructure/auth/auth.module' +import { UserPermissionModule } from '../infrastructure/permission/user/user.module' +import { KeycloakModule } from '../keycloak/keycloak.module' +import { NexusModule } from '../nexus/nexus.module' +import { RegistryModule } from '../registry/registry.module' +import { SonarqubeModule } from '../sonarqube/sonarqube.module' +import { VaultModule } from '../vault/vault.module' +import { ServiceMonitorController } from './service-monitor.controller' +import { ServiceMonitorService } from './service-monitor.service' + +@Module({ + imports: [ + CacheModule.register(), + AuthModule, + UserPermissionModule, + KeycloakModule, + ConditionalModule.registerWhen(ArgoCDModule, 'USE_ARGOCD'), + ConditionalModule.registerWhen(GitlabModule, 'USE_GITLAB'), + ConditionalModule.registerWhen(RegistryModule, 'USE_HARBOR'), + ConditionalModule.registerWhen(NexusModule, 'USE_NEXUS'), + ConditionalModule.registerWhen(SonarqubeModule, 'USE_SONARQUBE'), + ConditionalModule.registerWhen(VaultModule, 'USE_VAULT'), + ], + controllers: [ServiceMonitorController], + providers: [ServiceMonitorService], +}) +export class ServiceMonitorModule {} diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.spec.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.spec.ts new file mode 100644 index 0000000000..2c40ec2006 --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.spec.ts @@ -0,0 +1,182 @@ +import type { Mock } from 'vitest' +import type { DeepMockProxy } from 'vitest-mock-extended' +import { MonitorStatus } from '@cpn-console/shared' +import { CACHE_MANAGER } from '@nestjs/cache-manager' +import { Test } from '@nestjs/testing' +import { createCache } from 'cache-manager' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { mockDeep } from 'vitest-mock-extended' +import { ArgoCDHealthService } from '../argocd/argocd-health.service' +import { GitlabHealthService } from '../gitlab/gitlab-health.service' +import { KeycloakHealthService } from '../keycloak/keycloak-health.service' +import { NexusHealthService } from '../nexus/nexus-health.service' +import { RegistryHealthService } from '../registry/registry-health.service' +import { SonarqubeHealthService } from '../sonarqube/sonarqube-health.service' +import { VaultHealthService } from '../vault/vault-health.service' +import { INTERVAL_MS } from './service-monitor.constants' +import { ServiceMonitorService } from './service-monitor.service' + +describe('serviceMonitorService', () => { + describe('with every optional probe registered', () => { + let keycloak: DeepMockProxy + let argocd: DeepMockProxy + let gitlab: DeepMockProxy + let harbor: DeepMockProxy + let nexus: DeepMockProxy + let sonarqube: DeepMockProxy + let vault: DeepMockProxy + let service: ServiceMonitorService + + beforeEach(async () => { + keycloak = mockDeep() + argocd = mockDeep() + gitlab = mockDeep() + harbor = mockDeep() + nexus = mockDeep() + sonarqube = mockDeep() + vault = mockDeep() + keycloak.check.mockResolvedValue({ keycloak: { status: 'up', httpStatus: 200 } }) + argocd.check.mockResolvedValue({ argocd: { status: 'up', httpStatus: 200 } }) + gitlab.check.mockResolvedValue({ gitlab: { status: 'up', httpStatus: 200 } }) + harbor.check.mockResolvedValue({ harbor: { status: 'up' } }) + nexus.check.mockResolvedValue({ nexus: { status: 'up', httpStatus: 200 } }) + sonarqube.check.mockResolvedValue({ sonarqube: { status: 'up' } }) + vault.check.mockResolvedValue({ vault: { status: 'up', httpStatus: 200 } }) + const moduleRef = await Test.createTestingModule({ + providers: [ + ServiceMonitorService, + { provide: CACHE_MANAGER, useValue: createCache() }, + { provide: KeycloakHealthService, useValue: keycloak }, + { provide: ArgoCDHealthService, useValue: argocd }, + { provide: GitlabHealthService, useValue: gitlab }, + { provide: RegistryHealthService, useValue: harbor }, + { provide: NexusHealthService, useValue: nexus }, + { provide: SonarqubeHealthService, useValue: sonarqube }, + { provide: VaultHealthService, useValue: vault }, + ], + }).compile() + service = moduleRef.get(ServiceMonitorService) + }) + + it('aggregates every injected probe', async () => { + keycloak.check.mockResolvedValue({ keycloak: { status: 'down', httpStatus: 500 } }) + + const health = await service.refreshServiceHealth() + + expect(health).toEqual([ + { name: 'ArgoCD', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + { name: 'Gitlab', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + { name: 'Harbor', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + { name: 'Keycloak', status: MonitorStatus.ERROR, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: 'Service en erreur', cause: 'Service en erreur' }, + { name: 'Nexus', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + { name: 'SonarQube', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + { name: 'Vault', status: MonitorStatus.OK, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: MonitorStatus.OK }, + ]) + }) + + it('passes the SonarQube and Harbor degraded and failed verdicts through', async () => { + sonarqube.check.mockResolvedValue({ sonarqube: { status: 'up', message: 'Service dégradé' } }) + harbor.check.mockResolvedValue({ harbor: { status: 'down', message: 'Service en erreur' } }) + + const health = await service.refreshServiceHealth() + + expect(health).toContainEqual(expect.objectContaining({ name: 'SonarQube', status: MonitorStatus.WARNING, message: 'Service dégradé' })) + expect(health).toContainEqual(expect.objectContaining({ name: 'Harbor', status: MonitorStatus.ERROR, message: 'Service en erreur' })) + }) + + it('serves the last known state instead of re-probing on every read', async () => { + await service.refreshServiceHealth() + keycloak.check.mockClear() + + await service.getServiceHealth() + await service.getServiceHealth() + + expect(keycloak.check).not.toHaveBeenCalled() + }) + + it('always returns the cause', async () => { + keycloak.check.mockResolvedValue({ keycloak: { status: 'down', httpStatus: 503, message: 'connection refused' } }) + await service.refreshServiceHealth() + + expect(await service.getServiceHealth()).toContainEqual(expect.objectContaining({ name: 'Keycloak', cause: 'connection refused' })) + }) + + it('refreshServiceHealth re-probes every enabled service', async () => { + await service.refreshServiceHealth() + await service.refreshServiceHealth() + + expect(keycloak.check).toHaveBeenCalledTimes(2) + expect(sonarqube.check).toHaveBeenCalledTimes(2) + expect(harbor.check).toHaveBeenCalledTimes(2) + }) + + it('probes once on init', async () => { + service.onModuleInit() + await new Promise(resolve => setImmediate(resolve)) + + expect(keycloak.check).toHaveBeenCalledTimes(1) + }) + }) + + describe('without optional probes registered', () => { + let keycloak: { check: Mock<() => Promise> } + let service: ServiceMonitorService + + beforeEach(async () => { + keycloak = { check: vi.fn<() => Promise>() } + const moduleRef = await Test.createTestingModule({ + providers: [ + ServiceMonitorService, + { provide: CACHE_MANAGER, useValue: createCache() }, + { provide: KeycloakHealthService, useValue: keycloak }, + ], + }).compile() + service = moduleRef.get(ServiceMonitorService) + }) + + it('skips unregistered probes', async () => { + keycloak.check.mockResolvedValue({ keycloak: { status: 'up', httpStatus: 200 } }) + + const health = await service.refreshServiceHealth() + + expect(health.map(entry => entry.name)).toEqual(['Keycloak']) + }) + + it('serves the pending state until the first probe', async () => { + expect(await service.getServiceHealth()).toEqual([ + { name: 'Keycloak', status: MonitorStatus.UNKNOW, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: 'En attente d\'une première vérification', cause: 'App just started' }, + ]) + }) + + it('reports UNKNOW with the request error when the probe rejects', async () => { + keycloak.check.mockRejectedValue(new Error('boom')) + + const health = await service.refreshServiceHealth() + + expect(health).toEqual([ + { name: 'Keycloak', status: MonitorStatus.UNKNOW, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: 'Erreur lors la requête', cause: new Error('boom') }, + ]) + expect(await service.getServiceHealth()).toContainEqual(expect.objectContaining({ name: 'Keycloak', cause: new Error('boom') })) + }) + + it('reports UNKNOW when the probe payload has no status field', async () => { + keycloak.check.mockResolvedValue({ keycloak: {} }) + + const health = await service.refreshServiceHealth() + + expect(health).toEqual([ + { name: 'Keycloak', status: MonitorStatus.UNKNOW, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: 'Service en erreur' }, + ]) + }) + + it('reports ERROR with the probe detail message when status is not up', async () => { + keycloak.check.mockResolvedValue({ keycloak: { status: 'down', httpStatus: 503, message: 'connection refused' } }) + + const health = await service.refreshServiceHealth() + + expect(health).toEqual([ + { name: 'Keycloak', status: MonitorStatus.ERROR, interval: INTERVAL_MS, lastUpdateTimestamp: expect.any(Number), message: 'connection refused', cause: 'connection refused' }, + ]) + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.ts new file mode 100644 index 0000000000..ef3e7306c0 --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.service.ts @@ -0,0 +1,112 @@ +import type { MonitorInfos } from '@cpn-console/shared' +import type { OnModuleInit } from '@nestjs/common' +import type { Cache } from 'cache-manager' +import type { ProbeOutcome } from './service-monitor.utils' +import { MonitorStatus } from '@cpn-console/shared' +import { CACHE_MANAGER } from '@nestjs/cache-manager' +import { Inject, Injectable, Optional } from '@nestjs/common' +import { Cron, CronExpression } from '@nestjs/schedule' +import { ArgoCDHealthService } from '../argocd/argocd-health.service' +import { GitlabHealthService } from '../gitlab/gitlab-health.service' +import { KeycloakHealthService } from '../keycloak/keycloak-health.service' +import { NexusHealthService } from '../nexus/nexus-health.service' +import { RegistryHealthService } from '../registry/registry-health.service' +import { SonarqubeHealthService } from '../sonarqube/sonarqube-health.service' +import { VaultHealthService } from '../vault/vault-health.service' +import { INTERVAL_MS, PENDING_MESSAGE, REQUEST_ERROR_MESSAGE } from './service-monitor.constants' +import { fromHealthCheck } from './service-monitor.utils' + +export type ServiceHealth = MonitorInfos & { name: string } + +interface MonitoredService { + name: string + probe: () => Promise +} + +const CACHE_KEY = 'service-monitor:statuses' + +@Injectable() +export class ServiceMonitorService implements OnModuleInit { + private readonly interval: number = INTERVAL_MS + private readonly services: MonitoredService[] + + constructor( + @Inject(CACHE_MANAGER) private readonly cache: Cache, + @Inject(KeycloakHealthService) private readonly keycloak: KeycloakHealthService, + @Inject(ArgoCDHealthService) @Optional() private readonly argocd?: ArgoCDHealthService, + @Inject(GitlabHealthService) @Optional() private readonly gitlab?: GitlabHealthService, + @Inject(RegistryHealthService) @Optional() private readonly harbor?: RegistryHealthService, + @Inject(NexusHealthService) @Optional() private readonly nexus?: NexusHealthService, + @Inject(SonarqubeHealthService) @Optional() private readonly sonarqube?: SonarqubeHealthService, + @Inject(VaultHealthService) @Optional() private readonly vault?: VaultHealthService, + ) { + const { + argocd: argocdHealth, + gitlab: gitlabHealth, + harbor: harborHealth, + nexus: nexusHealth, + sonarqube: sonarqubeHealth, + vault: vaultHealth, + } = this + const probes: Array<[string, (() => Promise) | undefined]> = [ + ['ArgoCD', argocdHealth && (() => argocdHealth.check().then(result => fromHealthCheck(result, 'argocd')))], + ['Gitlab', gitlabHealth && (() => gitlabHealth.check().then(result => fromHealthCheck(result, 'gitlab')))], + ['Harbor', harborHealth && (() => harborHealth.check().then(result => fromHealthCheck(result, 'harbor')))], + ['Keycloak', () => this.keycloak.check().then(result => fromHealthCheck(result, 'keycloak'))], + ['Nexus', nexusHealth && (() => nexusHealth.check().then(result => fromHealthCheck(result, 'nexus')))], + ['SonarQube', sonarqubeHealth && (() => sonarqubeHealth.check().then(result => fromHealthCheck(result, 'sonarqube')))], + ['Vault', vaultHealth && (() => vaultHealth.check().then(result => fromHealthCheck(result, 'vault')))], + ] + + this.services = probes.flatMap(([name, probe]) => probe ? [{ name, probe }] : []) + } + + async getServiceHealth(): Promise { + const statuses = await Promise.all(this.services.map(async ({ name }) => { + const cached = await this.cache.get(`${CACHE_KEY}:${name}`) + return { name, ...(cached ?? this.pendingStatus()) } + })) + return statuses + } + + async refreshServiceHealth(): Promise { + await this.runAll() + return this.getServiceHealth() + } + + onModuleInit(): void { + void this.runAll() + } + + @Cron(CronExpression.EVERY_5_MINUTES) + private async runAll(): Promise { + await Promise.all(this.services.map(async (service) => { + try { + const outcome = await service.probe() + await this.writeStatus(service.name, { ...outcome, interval: this.interval, lastUpdateTimestamp: Date.now() }) + } catch (error) { + await this.writeStatus(service.name, { + interval: this.interval, + lastUpdateTimestamp: Date.now(), + status: MonitorStatus.UNKNOW, + message: REQUEST_ERROR_MESSAGE, + cause: error, + }) + } + })) + } + + private async writeStatus(name: string, status: MonitorInfos): Promise { + await this.cache.set(`${CACHE_KEY}:${name}`, status) + } + + private pendingStatus(): MonitorInfos { + return { + interval: this.interval, + lastUpdateTimestamp: Date.now(), + status: MonitorStatus.UNKNOW, + message: PENDING_MESSAGE, + cause: 'App just started', + } + } +} diff --git a/apps/server-nestjs/src/modules/service-monitor/service-monitor.utils.ts b/apps/server-nestjs/src/modules/service-monitor/service-monitor.utils.ts new file mode 100644 index 0000000000..e326d1237b --- /dev/null +++ b/apps/server-nestjs/src/modules/service-monitor/service-monitor.utils.ts @@ -0,0 +1,42 @@ +import { MonitorStatus } from '@cpn-console/shared' +import { isRecord } from '../../utils/record.utils' +import { ERROR_MESSAGE } from './service-monitor.constants' + +export interface ProbeOutcome { + status: MonitorStatus + message: string + cause?: unknown +} + +interface ProbeDetail { + status: string + message?: string + cause?: unknown +} + +function isProbeDetail(detail: unknown): detail is ProbeDetail { + return isRecord(detail) && 'status' in detail && typeof detail.status === 'string' +} + +export function fromHealthCheck(result: unknown, key: string): ProbeOutcome { + const detail = isRecord(result) ? result[key] : undefined + if (!isProbeDetail(detail)) { + return { status: MonitorStatus.UNKNOW, message: ERROR_MESSAGE } + } + + const up = detail.status === 'up' + if (up) { + return detail.cause !== undefined || detail.message !== undefined + ? { status: detail.cause !== undefined ? MonitorStatus.UNKNOW : MonitorStatus.WARNING, message: detail.message ?? MonitorStatus.WARNING } + : { status: MonitorStatus.OK, message: MonitorStatus.OK } + } + return { + status: MonitorStatus.ERROR, + message: detail.message ?? ERROR_MESSAGE, + cause: toCause(detail.cause ?? detail.message ?? ERROR_MESSAGE), + } +} + +export function toCause(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause) +} diff --git a/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.spec.ts b/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.spec.ts new file mode 100644 index 0000000000..0aa051f3e1 --- /dev/null +++ b/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.spec.ts @@ -0,0 +1,81 @@ +import type { ConfigType } from '@nestjs/config' +import { HealthIndicatorService } from '@nestjs/terminus' +import { Test } from '@nestjs/testing' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { mockDeep } from 'vitest-mock-extended' +import { sonarqubeConfigFactory } from '../../config/sonarqube.config' +import { SonarqubeHealthService } from './sonarqube-health.service' + +const sonarUrl = 'https://sonarqube.internal' +const healthUrl = `${sonarUrl}/api/system/health` + +const server = setupServer() + +describe('sonarqubeHealthService', () => { + let service: SonarqubeHealthService + + beforeAll(() => server.listen({ onUnhandledRequest: 'error' })) + + beforeEach(async () => { + const config = mockDeep>({ + apiToken: 'my-token', + url: sonarUrl, + internalUrl: undefined, + }) + + const module = await Test.createTestingModule({ + providers: [ + SonarqubeHealthService, + { provide: sonarqubeConfigFactory.KEY, useValue: config }, + HealthIndicatorService, + ], + }).compile() + + service = module.get(SonarqubeHealthService) + }) + afterEach(() => server.resetHandlers()) + afterAll(() => server.close()) + + it.each([ + ['GREEN', { status: 'up' }], + ['YELLOW', { status: 'up', message: 'Service dégradé' }], + ['RED', { status: 'down', message: 'Service en panne' }], + ])('maps SonarQube health %s', async (health, expected) => { + server.use(http.get(healthUrl, () => HttpResponse.json({ health, causes: [] }))) + + await expect(service.check()).resolves.toEqual({ sonarqube: expected }) + }) + + it('sends the raw api token under Bearer', async () => { + let authorization: string | undefined + server.use(http.get(healthUrl, ({ request }) => { + authorization = request.headers.get('authorization') ?? undefined + return HttpResponse.json({ health: 'GREEN', causes: [] }) + })) + + await expect(service.check()).resolves.toEqual({ sonarqube: { status: 'up' } }) + expect(authorization).toBe('Bearer my-token') + }) + + it('reports down when the API answers without a health field', async () => { + server.use(http.get(healthUrl, () => HttpResponse.json({}))) + + await expect(service.check()).resolves.toEqual({ sonarqube: { status: 'down', message: 'Erreur lors la requête' } }) + }) + + it('reports down when the API answers with a non-200 status', async () => { + server.use(http.get(healthUrl, () => new HttpResponse(null, { status: 503 }))) + + await expect(service.check()).resolves.toEqual({ sonarqube: { status: 'down', message: 'Erreur lors la requête' } }) + }) + + it('reports down with the cause when the request fails', async () => { + server.use(http.get(healthUrl, () => HttpResponse.error())) + + await expect(service.check()).resolves.toMatchObject({ + sonarqube: { status: 'down', message: 'Erreur lors la requête', cause: expect.any(Error) }, + }) + }) +}) diff --git a/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.ts b/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.ts index 388e0452a9..11816389d0 100644 --- a/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.ts +++ b/apps/server-nestjs/src/modules/sonarqube/sonarqube-health.service.ts @@ -2,6 +2,8 @@ import type { ConfigType } from '@nestjs/config' import { HttpStatus, Inject, Injectable } from '@nestjs/common' import { HealthIndicatorService } from '@nestjs/terminus' import { sonarqubeConfigFactory } from '../../config/sonarqube.config' +import { isRecord } from '../../utils/record.utils' +import { PLUGIN_NAME, REQUEST_ERROR_MESSAGE } from './sonarqube.constants' @Injectable() export class SonarqubeHealthService { @@ -10,18 +12,28 @@ export class SonarqubeHealthService { @Inject(HealthIndicatorService) private readonly healthIndicator: HealthIndicatorService, ) {} - async check(key: string) { - const indicator = this.healthIndicator.check(key) - const headers: Record = {} - const bearerToken = Buffer.from(`${this.sonarqubeConfig.apiToken}:`, 'utf-8').toString('base64') - headers.Authorization = `Bearer ${bearerToken}` + async check() { + const indicator = this.healthIndicator.check(PLUGIN_NAME) try { - const url = new URL('/api/system/health', this.sonarqubeConfig.internalUrl ?? this.sonarqubeConfig.url).toString() - const response = await fetch(url, { headers }) - if (response.status < HttpStatus.INTERNAL_SERVER_ERROR) return indicator.up({ httpStatus: response.status }) - return indicator.down({ httpStatus: response.status }) + const response = await fetch(this.healthUrl(), { headers: this.authHeaders() }) + if (response.status !== HttpStatus.OK) return indicator.down({ message: REQUEST_ERROR_MESSAGE }) + + const data: unknown = await response.json() + const health = isRecord(data) && typeof data.health === 'string' ? data.health : undefined + if (health === 'GREEN') return indicator.up() + if (health === 'YELLOW') return indicator.up({ message: 'Service dégradé' }) + if (health === 'RED') return indicator.down({ message: 'Service en panne' }) + return indicator.down({ message: REQUEST_ERROR_MESSAGE }) } catch (error) { - return indicator.down(error instanceof Error ? error.message : String(error)) + return indicator.down({ message: REQUEST_ERROR_MESSAGE, cause: error }) } } + + private healthUrl(): string { + return new URL('/api/system/health', this.sonarqubeConfig.internalUrl ?? this.sonarqubeConfig.url).toString() + } + + private authHeaders(): Record { + return { Authorization: `Bearer ${this.sonarqubeConfig.apiToken}` } + } } diff --git a/apps/server-nestjs/src/modules/sonarqube/sonarqube.constants.ts b/apps/server-nestjs/src/modules/sonarqube/sonarqube.constants.ts index 0935f737c6..66a5db7df9 100644 --- a/apps/server-nestjs/src/modules/sonarqube/sonarqube.constants.ts +++ b/apps/server-nestjs/src/modules/sonarqube/sonarqube.constants.ts @@ -54,3 +54,6 @@ export const SONARQUBE_PROJECT_QUALIFIER_PROJECT = 'TRK' export const SONARQUBE_PROJECT_QUALIFIER_SUB_VIEW = 'SVW' export const SONARQUBE_PROJECT_QUALIFIER_UNIT_TEST = 'UTS' export const SONARQUBE_PROJECT_QUALIFIER_VIEW = 'VW' + +// Health monitor messages +export const REQUEST_ERROR_MESSAGE = 'Erreur lors la requête' diff --git a/apps/server-nestjs/src/utils/record.utils.ts b/apps/server-nestjs/src/utils/record.utils.ts index 05418fcf86..0c6d57ec6d 100644 --- a/apps/server-nestjs/src/utils/record.utils.ts +++ b/apps/server-nestjs/src/utils/record.utils.ts @@ -1,3 +1,7 @@ export function hasEntries(record: Record): boolean { return Object.keys(record).length > 0 } + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} diff --git a/packages/shared/src/contracts/service-monitor.ts b/packages/shared/src/contracts/service-monitor.ts index c2b02e304b..6ec2d10b7a 100644 --- a/packages/shared/src/contracts/service-monitor.ts +++ b/packages/shared/src/contracts/service-monitor.ts @@ -23,7 +23,7 @@ export const serviceContract = contractInstance.router({ description: 'Retrieve services health with cause.', responses: { 200: ServiceHealthSchema.extend({ - cause: z.any().optional(), + cause: z.string().optional(), }).array(), 401: ErrorSchema, 403: ErrorSchema, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f8dc59f157..6573cc265d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -761,36 +761,15 @@ importers: apps/server-nestjs: dependencies: - '@cpn-console/argocd-plugin': - specifier: workspace:^ - version: link:../../plugins/argocd - '@cpn-console/gitlab-plugin': - specifier: workspace:^ - version: link:../../plugins/gitlab - '@cpn-console/harbor-plugin': - specifier: workspace:^ - version: link:../../plugins/harbor '@cpn-console/hooks': specifier: workspace:^ version: link:../../packages/hooks - '@cpn-console/keycloak-plugin': - specifier: workspace:^ - version: link:../../plugins/keycloak '@cpn-console/logger': specifier: workspace:^ version: link:../../packages/logger - '@cpn-console/nexus-plugin': - specifier: workspace:^ - version: link:../../plugins/nexus '@cpn-console/shared': specifier: workspace:^ version: link:../../packages/shared - '@cpn-console/sonarqube-plugin': - specifier: workspace:^ - version: link:../../plugins/sonarqube - '@cpn-console/vault-plugin': - specifier: workspace:^ - version: link:../../plugins/vault '@gitbeaker/core': specifier: catalog:runtime version: 40.6.0