feat(hot-reload): add polling mechanism MAPCO-9473

README.md

@@ -24,7 +24,11 @@ const configInstance = await config({
   configServerUrl: 'http://localhost:8080',
   schema: commonBoilerplateV4,
   version: 'latest',
-  offlineMode: false
+  offlineMode: false,
+  pollIntervalMs: 30000,
+  onChange: () => {
+    console.log('Configuration changed! The process will now restart...');
+  }
 });
 
 const port = configInstance.get('server.port');
@@ -36,14 +40,14 @@ This section describes the API provided by the package for interacting with the
 
 ### `ConfigInstance<T>`
 
-The `ConfigInstance` interface represents the your way to interact with the configuration. It provides methods to retrieve configuration values and parts.
-`T` is the typescript type associated with the chosen schema. it can be imported from the `@map-colonies/schemas` package.
+The `ConfigInstance` interface represents your way to interact with the configuration.
+`T` is the typescript type associated with the chosen schema. It can be imported from the `@map-colonies/schemas` package.
 
 #### Methods
 
 ##### `get<TPath extends string>(path: TPath): _.GetFieldType<T, TPath>`
 
-- **Description**: Retrieves the value at the specified path from the configuration object. Note that the type of returned object is based on the path in the schema.
+- **Description**: Retrieves the value at the specified path from the configuration object.
 - **Parameters**:
   - `path` (`TPath`): The path to the desired value.
 - **Returns**: The value at the specified path.
@@ -55,7 +59,7 @@ The `ConfigInstance` interface represents the your way to interact with the conf
 
 ##### `getConfigParts(): { localConfig: object; config: object; envConfig: object }`
 
-- **Description**: Retrieves different parts of the configuration object before being merged and validated. Useful for debugging.
+- **Description**: Retrieves different parts of the configuration object before being merged and validated.
 - **Returns**: An object containing the `localConfig`, `config`, and `envConfig` parts of the configuration.
   - `localConfig`: The local configuration object.
   - `config`: The remote configuration object.
@@ -71,6 +75,9 @@ The `ConfigInstance` interface represents the your way to interact with the conf
 - **Parameters**:
   - `registry` (`promClient.Registry`): The prometheus registry to use for the metrics.
 
+##### `stop(): void`
+- **Description**: Stops any background processes (like hot-reloading polling). Use this during application teardown or in tests to prevent memory leaks and hanging processes.
+
 # Configuration Options
 
 This package allows you to configure various options for loading and managing configurations. Below are the available options and their descriptions.
@@ -121,6 +128,18 @@ This package allows you to configure various options for loading and managing co
 - **Default**: `./config`
 - **Description**: The path to the local configuration folder.
 
+### `pollIntervalMs`
+- **Type**: `number`
+- **Optional**: `true`
+- **Default**: `30000`
+- **Description**: The polling interval in milliseconds for hot-reloading.
+- **Environment Variable**: `CONFIG_POLL_INTERVAL_MS`
+
+### `onChange`
+- **Type**: `(config: T) => void | Promise<void>`
+- **Optional**: `true`
+- **Description**: A callback function triggered when a configuration change is detected.
+
 ## Environment Variable Configuration
 
 The following environment variables can be used to configure the options:
@@ -130,6 +149,7 @@ The following environment variables can be used to configure the options:
 - `CONFIG_SERVER_URL`: Sets the `configServerUrl` option.
 - `CONFIG_OFFLINE_MODE`: Sets the `offlineMode` option.
 - `CONFIG_IGNORE_SERVER_IS_OLDER_VERSION_ERROR`: Sets the `ignoreServerIsOlderVersionError` option.
+- `CONFIG_POLL_INTERVAL_MS`: Sets the `pollIntervalMs` option.
 
 ## Configuration Merging and Validation
 
@@ -143,6 +163,7 @@ The package supports merging configurations from multiple sources (local, remote
 
 1. The remote configuration is fetched from the server specified by the `configServerUrl` option.
 2. If the `version` is set to `'latest'`, the latest version of the configuration is fetched. Otherwise, the specified version is fetched.
+3. **Continuous Polling:** The SDK continuously polls the server using HTTP ETags (`If-None-Match`). When a `200 OK` is received (indicating a change), the `onChange` callback is triggered (if provided), and the process is then terminated to allow for a fresh start with the new configuration. `304 Not Modified` responses are silently ignored.
 
 ### Environment Variables
 
@@ -164,7 +185,7 @@ If the value of the `x-env-format` key is `json`, the environment variable value
 1. After merging, the final configuration is validated against the defined schema using ajv.
 2. The validation ensures that all required properties are present, and the types and values of properties conform to the schema.
 3. Any default value according to the schema is added to the final object.
-4. If the validation fails, an error is thrown, indicating the invalid properties and their issues.
+4. If the validation fails, an error is thrown (for initial boot).
 
 
 # Error handling

src/config.ts

@@ -15,6 +15,7 @@ import { LOCAL_SCHEMAS_PACKAGE_VERSION } from './constants';
 import { createConfigError } from './errors';
 import { initializeMetrics as initializeMetricsInternal } from './metrics';
 import { deepFreeze } from './utils/helpers';
+import { ChangeDetector } from './rollout/ChangeDetector';
 
 const debug = createDebug('config');
 
@@ -25,22 +26,59 @@ const semverSatisfies = '2.x';
 /**
  * Retrieves the configuration based on the provided options.
  *
+ * If `offlineMode` is not enabled, the SDK starts a background
+ * polling mechanism. When a configuration change is detected, the SDK
+ * will execute the `onChange` callback (if provided).
+ * Depending on the `terminatePod` option (default `true`), it will either
+ * stop polling and wait for the user to terminate the process, or continue polling.
+ *
  * @template T - The type of the configuration schema.
  * @param {ConfigOptions<T>} options - The options for retrieving the configuration.
- * @returns {Promise<ConfigInstance<T>>} - A promise that resolves to the configuration object.
+ * @returns {Promise<ConfigInstance<T[typeof typeSymbol]>>} - A promise that resolves to the configuration object.
  */
 export async function config<T extends { [typeSymbol]: unknown; $id: string }>(
   options: ConfigOptions<T>
 ): Promise<ConfigInstance<T[typeof typeSymbol]>> {
   // handle package options
   debug('config called with options: %j', { ...options, schema: options.schema.$id });
-  const { schema: baseSchema, metricsRegistry, ...unvalidatedOptions } = options;
-  const { configName, offlineMode, version, ignoreServerIsOlderVersionError } = initializeOptions(unvalidatedOptions);
+  const { schema: baseSchema, metricsRegistry, onChange, ...unvalidatedOptions } = options;
+  const initOptions = initializeOptions(unvalidatedOptions);
+  const { configName, offlineMode, version, ignoreServerIsOlderVersionError } = initOptions;
 
-  let remoteConfig: object | T = {};
+  // Load Local and Env Configs First (Independent of remote state)
+  const dereferencedSchema = await loadSchema(baseSchema);
+  const localConfig = configPkg.util.loadFileConfigs(options.localConfigPath) as { [key: string]: unknown };
+  debug('local config: %j', localConfig);
+  const envConfig = getEnvValues(dereferencedSchema);
+  debug('env config: %j', envConfig);
 
+  /**
+   * Function to merge local, remote, and environment configs and validate the merged result against the schema.
+   * The precedence order for merging is: local < remote < environment.
+   */
+  function mergeAndValidate(remoteConfig: object | T): unknown {
+    const mergedConfig = deepmerge.all([localConfig, remoteConfig, envConfig], { arrayMerge });
+    debug('merged config: %j', mergedConfig);
+
+    // validate the merged config
+    const [errors, validatedConfig] = validate(ajvConfigValidator, dereferencedSchema, mergedConfig);
+    if (errors) {
+      debug('config validation error: %j', errors);
+      throw createConfigError('configValidationError', 'Config validation error', errors);
+    }
+
+    debug('freezing validated config');
+    // freeze the merged config so it can't be modified by the package user and to ensure that the config instance always returns the same reference for the same config, which is important for change detection and to prevent unnecessary re-renders in case the config is used in a React application and the user is using the getConfigParts method to get the config and pass it to their components
+    deepFreeze(validatedConfig);
+    return validatedConfig;
+  }
+
+  let changeDetector: ChangeDetector | undefined = undefined;
+  let remoteConfig: object | T = {};
   let serverConfigResponse: Config | undefined = undefined;
-  // handle remote config
+  let validatedConfig: ReturnType<typeof mergeAndValidate>;
+
+  // Handle Remote Config and Polling
   if (offlineMode !== true) {
     debug('handling fetching remote data');
     // check if the server is using an older version of the schemas package
@@ -70,8 +108,10 @@ export async function config<T extends { [typeSymbol]: unknown; $id: string }>(
       );
     }
 
-    // get the remote config
-    serverConfigResponse = await getRemoteConfig(configName, options.schema.$id, version);
+    // get the initial remote config
+    const remoteResponse = await getRemoteConfig(configName, options.schema.$id, version);
+    serverConfigResponse = remoteResponse.config!;
+    const currentEtag = remoteResponse.etag;
 
     if (serverConfigResponse.schemaId !== baseSchema.$id) {
       debug('schema version mismatch. local: %s, remote: %s', baseSchema.$id, serverConfigResponse.schemaId);
@@ -86,32 +126,18 @@ export async function config<T extends { [typeSymbol]: unknown; $id: string }>(
     }
 
     remoteConfig = serverConfigResponse.config;
-  }
-  debug('remote config: %j', remoteConfig);
+    debug('remote config: %j', remoteConfig);
 
-  const dereferencedSchema = await loadSchema(baseSchema);
+    validatedConfig = mergeAndValidate(remoteConfig);
 
-  const localConfig = configPkg.util.loadFileConfigs(options.localConfigPath) as { [key: string]: unknown };
-  debug('local config: %j', localConfig);
-
-  const envConfig = getEnvValues(dereferencedSchema);
-  debug('env config: %j', envConfig);
-
-  // merge all the configs into one object with the following priority: localConfig < remoteConfig < envConfig
-  const mergedConfig = deepmerge.all([localConfig, remoteConfig, envConfig], { arrayMerge });
-  debug('merged config: %j', mergedConfig);
-
-  // validate the merged config
-  const [errors, validatedConfig] = validate(ajvConfigValidator, dereferencedSchema, mergedConfig);
-  if (errors) {
-    debug('config validation error: %j', errors);
-    throw createConfigError('configValidationError', 'Config validation error', errors);
+    // Setup polling
+    changeDetector = new ChangeDetector(baseSchema.$id, initOptions, currentEtag, onChange);
+    changeDetector.start();
+  } else {
+    // If offline, bypass remote and just merge local/env
+    validatedConfig = mergeAndValidate({});
   }
 
-  debug('freezing validated config');
-  // freeze the merged config so it can't be modified by the package user
-  deepFreeze(validatedConfig);
-
   function get<TPath extends string>(path: TPath): GetFieldType<T[typeof typeSymbol], TPath> {
     debug('get called with path: %s', path);
     return lodash.get(validatedConfig as (typeof baseSchema)[typeof typeSymbol], path);
@@ -140,5 +166,12 @@ export async function config<T extends { [typeSymbol]: unknown; $id: string }>(
     initializeMetricsInternal(registry, baseSchema.$id, serverConfigResponse?.version);
   }
 
-  return { get, getAll, getConfigParts, getResolvedOptions, initializeMetrics };
+  function stop(): void {
+    debug('stop called');
+    if (changeDetector) {
+      changeDetector.stop();
+    }
+  }
+
+  return { get, getAll, getConfigParts, getResolvedOptions, initializeMetrics, stop };
 }

src/httpClient.ts

@@ -15,10 +15,10 @@ async function createHttpErrorPayload(res: Dispatcher.ResponseData): Promise<Con
   };
 }
 
-async function requestWrapper(url: string, query?: Record<string, unknown>): Promise<Dispatcher.ResponseData> {
+async function requestWrapper(url: string, options: Parameters<typeof request<null>>[1] = undefined): Promise<Dispatcher.ResponseData> {
   debug('Making request to %s', url);
   try {
-    const res = await request(url, { query });
+    const res = await request(url, options);
     if (res.statusCode > statusCodes.NOT_FOUND) {
       debug('Failed to fetch config. Status code: %d', res.statusCode);
       throw createConfigError('httpResponseError', 'Failed to fetch', await createHttpErrorPayload(res));
@@ -33,12 +33,25 @@ async function requestWrapper(url: string, query?: Record<string, unknown>): Pro
   }
 }
 
-export async function getRemoteConfig(configName: string, schemaId: string, version: number | 'latest'): Promise<Config> {
+export async function getRemoteConfig(
+  configName: string,
+  schemaId: string,
+  version: number | 'latest',
+  etag?: string
+): Promise<{ config: Config | null; etag: string }> {
   debug('Fetching remote config %s@%s', configName, version);
   const { configServerUrl } = getOptions();
   const url = `${configServerUrl}/config/${configName}/${version}`;
 
-  const res = await requestWrapper(url, { shouldDereference: true, schemaId });
+  const headers = etag !== undefined ? { 'If-None-Match': etag } : undefined;
+  const queryParams = { schemaId, shouldDereference: 'true' };
+
+  const res = await requestWrapper(url, { query: queryParams, headers });
+
+  if (res.statusCode === statusCodes.NOT_MODIFIED) {
+    debug('Config was not modified');
+    return { config: null, etag: etag! };
+  }
 
   if (res.statusCode === statusCodes.BAD_REQUEST) {
     debug('Invalid request to getConfig');
@@ -51,7 +64,7 @@ export async function getRemoteConfig(configName: string, schemaId: string, vers
   }
   debug('Config fetched successfully');
 
-  return (await res.body.json()) as Config;
+  return { config: (await res.body.json()) as Config, etag: res.headers.etag as string };
 }
 
 export async function getServerCapabilities(): Promise<ServerCapabilities> {

src/options.ts

@@ -11,6 +11,8 @@ const defaultOptions: BaseOptions = {
   configName: PACKAGE_NAME,
   configServerUrl: 'http://localhost:8080',
   version: 'latest',
+  pollIntervalMs: 30000,
+  terminatePod: true,
 };
 
 const envOptions: Partial<Record<keyof BaseOptions, string>> = {
@@ -19,6 +21,8 @@ const envOptions: Partial<Record<keyof BaseOptions, string>> = {
   version: process.env.CONFIG_VERSION,
   offlineMode: process.env.CONFIG_OFFLINE_MODE,
   ignoreServerIsOlderVersionError: process.env.CONFIG_IGNORE_SERVER_IS_OLDER_VERSION_ERROR,
+  pollIntervalMs: process.env.CONFIG_POLL_INTERVAL_MS,
+  terminatePod: process.env.CONFIG_TERMINATE_POD,
 };
 
 // in order to merge correctly the keys should not exist, undefined is not enough

src/rollout/ChangeDetector.ts

@@ -0,0 +1,75 @@
+import { getRemoteConfig } from '../httpClient';
+import { BaseOptions } from '../types';
+import { createDebug } from '../utils/debug';
+import { isConfigError } from '../errors';
+
+const debug = createDebug('changeDetector');
+
+/**
+ * Class responsible for detecting changes in the remote configuration by periodically polling the server and comparing ETags.
+ * If a change is detected, it invokes the provided callback with the new configuration.
+ */
+export class ChangeDetector {
+  private currentEtag: string;
+  private timer?: NodeJS.Timeout;
+
+  public constructor(
+    private readonly schemaId: string,
+    private readonly options: BaseOptions,
+    initialEtag: string,
+    private readonly onConfigUpdate?: () => void | Promise<void>
+  ) {
+    this.currentEtag = initialEtag;
+  }
+
+  public start(): void {
+    const interval = this.options.pollIntervalMs;
+    debug('Starting change detector with interval %d ms', interval);
+
+    this.timer = setInterval(() => {
+      this.poll().catch((err) => {
+        debug('Error during polling: %s', (err as Error).message);
+      });
+    }, interval);
+  }
+
+  public stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+  }
+
+  private async poll(): Promise<void> {
+    debug('Polling config %s@%s with etag %s', this.options.configName, this.options.version, this.currentEtag);
+
+    const response = await getRemoteConfig(this.options.configName, this.schemaId, this.options.version, this.currentEtag);
+
+    if (response.config === null) {
+      debug('No config changes detected');
+      return;
+    }
+
+    debug('Config change detected. Stopping polling. New etag: %s', response.etag);
+    this.stop();
+    try {
+      if (this.onConfigUpdate) {
+        await this.onConfigUpdate();
+      }
+    } catch (err) {
+      if (isConfigError(err, 'httpResponseError') || isConfigError(err, 'httpGeneralError')) {
+        debug('Error during onChange callback: %s', err.message);
+      } else {
+        debug('Error during onChange callback: %s', err instanceof Error ? err.message : String(err));
+      }
+    } finally {
+      if (this.options.terminatePod) {
+        debug('Pod termination is expected by the user.');
+      } else {
+        this.currentEtag = response.etag;
+        debug('Pod termination is not expected.');
+        this.start();
+      }
+    }
+  }
+}