add decode utility method and use it as an option for the reduceToText functionality

Author: haschek

CHANGELOG.md

@@ -54,6 +54,7 @@ This is a major release, and it might be not compatible with your current usage
     -   `colorCalculateDistance()`: calculates the difference between 2 colors using the simple CIE76 formula
     -   `textToColorHash()`: calculates a color from a text string
     -   `reduceToText`: shrinks HTML content and React elements to plain text, used for `<TextReducer />`
+    -   `decodeHtmlEntities`: decode a string of HTML text, map HTML entities back to UTF-8 char
 -   SCSS color functions
     -   `eccgui-color-var`: returns a var of a custom property used for palette color
     -   `eccgui-color-mix`: mix 2 colors in `srgb`, works with all types of color values and CSS custom properties

src/common/index.ts

@@ -1,3 +1,5 @@
+import { decode } from "he";
+
 import { invisibleZeroWidthCharacters } from "./utils/characters";
 import { colorCalculateDistance } from "./utils/colorCalculateDistance";
 import decideContrastColorValue from "./utils/colorDecideContrastvalue";
@@ -6,7 +8,8 @@ import getColorConfiguration from "./utils/getColorConfiguration";
 import { getScrollParent } from "./utils/getScrollParent";
 import { getGlobalVar, setGlobalVar } from "./utils/globalVars";
 import { openInNewTab } from "./utils/openInNewTab";
-import { reduceToText } from "./utils/reduceToText"
+import { reduceToText } from "./utils/reduceToText";
+export type { DecodeOptions as DecodeHtmlEntitiesOptions } from "he";
 export type { IntentTypes as IntentBaseTypes } from "./Intent";
 
 export const utils = {
@@ -20,5 +23,6 @@ export const utils = {
     getScrollParent,
     getEnabledColorsFromPalette,
     textToColorHash,
-    reduceToText
+    reduceToText,
+    decodeHtmlEntities: decode,
 };

src/common/utils/reduceToText.tsx

@@ -3,19 +3,20 @@ import { renderToString } from "react-dom/server";
 import * as ReactIs from "react-is";
 
 import { TextReducerProps } from "./../../components/TextReducer/TextReducer";
+import { DecodeHtmlEntitiesOptions, utils } from "./../";
 
 export interface ReduceToTextFuncType {
     (
         /**
          *  Component or text to reduce HTML markup content to plain text.
          */
         input: React.ReactNode | React.ReactNode[] | string,
-        options?: Pick<TextReducerProps, "maxNodes" | "maxLength">
+        options?: Pick<TextReducerProps, "maxNodes" | "maxLength" | "decodeHtmlEntities" | "decodeHtmlEntitiesOptions">
     ): string;
 }
 
 export const reduceToText: ReduceToTextFuncType = (input, options) => {
-    const { maxNodes, maxLength } = options || {};
+    const { maxNodes, maxLength, decodeHtmlEntities } = options || {};
     const content: React.ReactNode | React.ReactNode[] = input;
     let nodeCount = 0;
 
@@ -46,6 +47,30 @@ export const reduceToText: ReduceToTextFuncType = (input, options) => {
     // Basic HTML cleanup
     text = text.replace(/<[^\s][^>]*>/g, "").replace(/\n/g, " ");
 
+    if (decodeHtmlEntities) {
+        const decodeDefaultOptions = {
+            isAttributeValue: true,
+            strict: true,
+        } as DecodeHtmlEntitiesOptions;
+        let decodeErrors = 0;
+        // we decode in pieces to some error tolerance even in strict mode
+        text = text
+            .split(" ")
+            .map((value) => {
+                try {
+                    return utils.decodeHtmlEntities(value, { ...decodeDefaultOptions });
+                } catch {
+                    decodeErrors++;
+                    return value;
+                }
+            })
+            .join(" ");
+        if (decodeErrors > 0) {
+            // eslint-disable-next-line no-console
+            console.warn(`${decodeErrors} parse error(s) for decodeHtmlEntities, return un-decoded text`, text);
+        }
+    }
+
     if (typeof maxLength === "number") {
         text = text.slice(0, maxLength);
     }

src/components/TextReducer/TextReducer.stories.tsx

@@ -18,8 +18,9 @@ Default.args = {
         <LoremIpsum p={1} avgSentencesPerParagraph={1} random={false} />,
         "Simple text with URL http://example.com/ that should not get parsed.",
         "a < b to test equations in text like b > a.",
+        `Something with a "quote" in it.`,
         <>
-            <Markdown>{`* This\n* is\n* a\n* list\n\nwritten in Markdown.`}</Markdown>
+            <Markdown>{`* This\n* is\n* a\n* list\n\nwritten in Markdown\n* containing a few HTML 'entities' & "quotes".`}</Markdown>
             <HtmlContentBlock>
                 <h1>Block with sub elements</h1>
                 <LoremIpsum p={3} avgSentencesPerParagraph={3} random={false} />

src/components/TextReducer/TextReducer.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 
-import { reduceToText } from "../../common/utils/reduceToText";
+import { DecodeHtmlEntitiesOptions, utils } from "../../common";
 import { CLASSPREFIX as eccgui } from "../../configuration/constants";
 
 import { OverflowText, OverflowTextProps } from "./../Typography";
@@ -24,6 +24,17 @@ export interface TextReducerProps extends Pick<React.HTMLAttributes<HTMLElement>
      * Specify more `OverflowText` properties used when `useOverflowTextWrapper` is set to `true`.
      */
     overflowTextProps?: Omit<OverflowTextProps, "passDown">;
+    /**
+     * If you transform HTML markup to text then the result could contain HTML entity encoded strings.
+     * By enabling this option they are decoded back to it's original char.
+     */
+    decodeHtmlEntities?: boolean;
+    /**
+     * Set the options used to decode the html entities, if `decodeHtmlEntities` is enabled.
+     * Internally we use `he` library, see their [documentation on decode options](https://www.npmjs.com/package/he#hedecodehtml-options).
+     * If not used we use `{ isAttributeValue: true, strict: true }` as default value.
+     */
+    decodeHtmlEntitiesOptions?: DecodeHtmlEntitiesOptions;
 }
 
 /**
@@ -32,16 +43,15 @@ export interface TextReducerProps extends Pick<React.HTMLAttributes<HTMLElement>
  */
 export const TextReducer = ({
     children,
-    maxNodes,
-    maxLength,
     useOverflowTextWrapper,
     overflowTextProps,
+    ...reduceToTextOptions
 }: TextReducerProps) => {
     if (typeof children === "undefined") {
         return <></>;
     }
 
-    const shrinkedContent = reduceToText(children, { maxLength, maxNodes });
+    const shrinkedContent = utils.reduceToText(children, reduceToTextOptions);
 
     return useOverflowTextWrapper ? (
         <OverflowText