[FEATURE] DPL-140: Readability score in RTE overlay

.github/workflows/publish.yml

@@ -53,6 +53,14 @@ jobs:
       - name: Install tailor
         run: composer global require typo3/tailor --prefer-dist --no-progress --no-suggest
 
+      # Installs the bundled contrib libraries into `contrib/Libraries` so they are
+      # packaged into the TER artifact for classic mode installations. The committed
+      # `contrib/composer.lock` is resolved with `--prefer-lowest` (enforced by the
+      # `checkContribComposer` integration check), so `install` ships the lowest
+      # supported library versions.
+      - name: Install contrib libraries
+        run: composer install -d contrib --no-dev
+
       # Note that step will fail when `env.version` does not match the `ext_emconf.php` version.
       - name: Create local TER package upload artifact
         env:

.github/workflows/testcore13.yml

@@ -34,6 +34,9 @@ jobs:
       - name: "Ensure UTF-8 files do not contain BOM"
         run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkBom"
 
+      - name: "Verify bundled contrib library composer constraint and lock"
+        run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkContribComposer"
+
 #      - name: "Test .rst files for integrity"
 #        run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkRst"
 

.gitignore

@@ -3,7 +3,7 @@
 .Build
 !.Build/Web/.gitkeep
 .ddev/docker-compose.tempfs.yaml
-composer.lock
+/composer.lock
 composer.json.testing
 composer.json.orig
 composer.json.pretty
@@ -22,3 +22,4 @@ Build/node_modules/
 vendor
 tailor-version-artefact/
 tailor-version-upload/
+/contrib/Libraries/

Build/Scripts/checkContribComposer.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+#
+# Integration check for the bundled contrib library shipped in `contrib/Libraries`
+# for classic mode / TER installations.
+#
+# It ensures that:
+#   1. the version constraint of the bundled package is identical in the root
+#      `composer.json` and in `contrib/composer.json`, and
+#   2. `contrib/composer.lock` is up to date and resolved with `--prefer-lowest`,
+#      so the lowest supported library version is the one shipped.
+#
+# Run via: Build/Scripts/runTests.sh -s checkContribComposer
+#
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)"
+cd "${ROOT_DIR}"
+
+PACKAGE="org_heigl/hyphenator"
+LOCK="contrib/composer.lock"
+LOCK_BACKUP="contrib/composer.lock.checkorig"
+
+fail() {
+    echo "ERROR: $*" >&2
+    exit 1
+}
+
+echo "[1/2] Checking '${PACKAGE}' constraint parity between composer.json and contrib/composer.json ..."
+ROOT_CONSTRAINT="$(php -r '$d = json_decode(file_get_contents("composer.json"), true); echo $d["require"][$argv[1]] ?? "";' "${PACKAGE}")"
+CONTRIB_CONSTRAINT="$(php -r '$d = json_decode(file_get_contents("contrib/composer.json"), true); echo $d["require"][$argv[1]] ?? "";' "${PACKAGE}")"
+
+[ -n "${ROOT_CONSTRAINT}" ] || fail "'${PACKAGE}' is not required in composer.json"
+[ -n "${CONTRIB_CONSTRAINT}" ] || fail "'${PACKAGE}' is not required in contrib/composer.json"
+if [ "${ROOT_CONSTRAINT}" != "${CONTRIB_CONSTRAINT}" ]; then
+    echo "  composer.json:         ${ROOT_CONSTRAINT}" >&2
+    echo "  contrib/composer.json: ${CONTRIB_CONSTRAINT}" >&2
+    fail "'${PACKAGE}' constraint differs between composer.json and contrib/composer.json"
+fi
+echo "  OK: both pin '${PACKAGE}' to '${ROOT_CONSTRAINT}'"
+
+echo "[2/2] Checking ${LOCK} is up to date (--prefer-lowest) ..."
+cp "${LOCK}" "${LOCK_BACKUP}"
+trap 'mv -f "${LOCK_BACKUP}" "${LOCK}" 2>/dev/null || true' EXIT
+composer update --prefer-lowest --no-install --no-interaction --no-progress -d contrib >/dev/null
+if ! diff -q "${LOCK}" "${LOCK_BACKUP}" >/dev/null; then
+    echo "--- committed ${LOCK}" >&2
+    echo "+++ freshly resolved (--prefer-lowest)" >&2
+    diff "${LOCK_BACKUP}" "${LOCK}" >&2 || true
+    fail "${LOCK} is out of date. Regenerate it with: composer update --prefer-lowest -d contrib"
+fi
+echo "  OK: ${LOCK} matches contrib/composer.json (--prefer-lowest)"
+
+echo "SUCCESS"

Build/Scripts/runTests.sh

@@ -147,6 +147,7 @@ Options:
             - checkBom: check UTF-8 files do not contain BOM
             - checkExceptionCodes: Check for duplicate exception codes
             - checkRst: test .rst files for integrity
+            - checkContribComposer: verify bundled contrib library constraint parity and prefer-lowest lock freshness
             - checkTestMethodsPrefix: check tests methods do not start with "test"
             - clean: clean up build, cache and testing related files and folders
             - cleanCache: clean up cache related files and folders
@@ -472,6 +473,11 @@ case ${TEST_SUITE} in
         ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name cgl-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"
         SUITE_EXIT_CODE=$?
         ;;
+    checkContribComposer)
+        COMMAND="Build/Scripts/checkContribComposer.sh"
+        ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-contrib-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"
+        SUITE_EXIT_CODE=$?
+        ;;
     checkRst)
         COMMAND="php -dxdebug.mode=off Build/Scripts/validateRstFiles.php"
         ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name cgl-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}"

Classes/Controller/ReadabilityController.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\DeeplWrite\Controller;
+
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use TYPO3\CMS\Backend\Attribute\AsController;
+use TYPO3\CMS\Core\Http\JsonResponse;
+use WebVision\DeeplWrite\Readability\ReadabilityCalculatorFactory;
+
+/**
+ * @internal
+ * This class is meant to be used within the DeepL write extension and therefore
+ * no public API. Endpoints can change without further information.
+ */
+#[AsController]
+final class ReadabilityController
+{
+    public function __construct(
+        private readonly ReadabilityCalculatorFactory $factory,
+    ) {
+    }
+
+    public function calculate(ServerRequestInterface $request): ResponseInterface
+    {
+        $data = $request->getParsedBody();
+        $data = is_array($data) ? $data : [];
+        $language = (string)($data['language'] ?? '');
+        $text = strip_tags((string)($data['text'] ?? ''));
+
+        try {
+            $result = $this->factory->fromLanguage($language)->calculateReadability($text);
+        } catch (\InvalidArgumentException) {
+            // No readability calculator is registered for the requested language,
+            // or the text contained no countable words. Report "no score" instead
+            // of failing the request, so the editor overlay keeps working.
+            return new JsonResponse(['score' => null]);
+        }
+
+        return new JsonResponse($result->jsonSerialize());
+    }
+}

Classes/Readability/Calculator/AbstractReadabilityCalculator.php

@@ -0,0 +1,118 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\DeeplWrite\Readability\Calculator;
+
+use Org\Heigl\Hyphenator\Hyphenator;
+use WebVision\DeeplWrite\Readability\ReadabilityCalculatorInterface;
+use WebVision\DeeplWrite\Readability\Result\ReadabilityResult;
+
+/**
+ * Base class for the Flesch reading-ease calculators.
+ *
+ * It provides the shared text metrics (sentences, words, syllables and
+ * characters) and the overall calculation flow. Concrete calculators only need
+ * to declare the language they serve, the hyphenation locale used for syllable
+ * counting and the language specific Flesch reading-ease formula in
+ * {@see self::calculateScore()}.
+ */
+abstract class AbstractReadabilityCalculator implements ReadabilityCalculatorInterface
+{
+    /**
+     * Language tag (matched by its primary subtag) this calculator serves.
+     */
+    protected const LANGUAGE = 'not-supported';
+
+    /**
+     * Locale used to load the org_heigl/hyphenator hyphenation patterns for
+     * syllable counting. Must match a dictionary shipped by the library, e.g.
+     * "de_DE", "en_US", "fr", "it_IT" or "pt_BR".
+     */
+    protected const HYPHENATION_LOCALE = 'en_US';
+
+    protected const SENTENCE_SPLIT = '/([!\.\?] )/';
+    protected const HYPHENATED_SPLIT = '/([(\s)+!\.\?|])/';
+
+    final public function calculateReadability(string $text): ReadabilityResult
+    {
+        $sentences = $this->countSentences($text);
+        $words = $this->countWords($text);
+        $syllables = $this->countSyllables($text);
+        $characters = $this->countCharacters($text);
+
+        if ($words <= 0) {
+            throw new \InvalidArgumentException(
+                'Readability can not be calculated for a text without countable words.',
+                1783350001
+            );
+        }
+        if ($sentences <= 0) {
+            $sentences = 1;
+        }
+
+        // Very short or very simple texts can push the formula above 100. As
+        // 100 is the defined maximum ("very easy to read") the score is capped
+        // here; this is a known property of the Flesch formulas and acceptable
+        // for the quick overview this feature provides.
+        $score = min(
+            $this->calculateScore($words / $sentences, $syllables / $words),
+            100.0,
+        );
+
+        return new ReadabilityResult($text, $sentences, $words, $syllables, $characters, $score);
+    }
+
+    /**
+     * Calculate the language specific Flesch reading-ease score.
+     *
+     * @param float $averageSentenceLength number of words / number of sentences (ASL)
+     * @param float $averageSyllablesPerWord number of syllables / number of words (ASW)
+     */
+    abstract protected function calculateScore(float $averageSentenceLength, float $averageSyllablesPerWord): float;
+
+    final protected function countSentences(string $text): int
+    {
+        $sentences = preg_split(self::SENTENCE_SPLIT, $text);
+        if ($sentences === false) {
+            return 0;
+        }
+        return count($sentences);
+    }
+
+    protected function countWords(string $text): int
+    {
+        // Count words as sequences of unicode letters, keeping intra-word
+        // apostrophes and hyphens together (e.g. "don't", "arc-en-ciel").
+        // Unlike str_word_count() this handles the accented characters used by
+        // languages such as French, Italian, Portuguese, Spanish and German.
+        return (int)preg_match_all('/\p{L}+(?:[\x27\x{2019}-]\p{L}+)*/u', $text);
+    }
+
+    final protected function countSyllables(string $text): int
+    {
+        $hyphenator = new Hyphenator();
+        $options = $hyphenator->getOptions();
+        $options->setDefaultLocale(static::HYPHENATION_LOCALE);
+        $options->setHyphen('|');
+        $result = $hyphenator->hyphenate($text);
+        if (!is_string($result)) {
+            return 0;
+        }
+        $splitted = preg_split(self::HYPHENATED_SPLIT, $result);
+        if ($splitted === false) {
+            return 0;
+        }
+        return count($splitted);
+    }
+
+    protected function countCharacters(string $text): int
+    {
+        return mb_strlen($text);
+    }
+
+    public function getLanguage(): string
+    {
+        return static::LANGUAGE;
+    }
+}

Classes/Readability/Calculator/FleschKincaidEnglish.php

@@ -0,0 +1,33 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\DeeplWrite\Readability\Calculator;
+
+use Symfony\Component\DependencyInjection\Attribute\AutoconfigureTag;
+
+/**
+ * Flesch reading-ease score for English, using the original Flesch (1948)
+ * formula:
+ *
+ * FRE = 206.835 - (1.015 * ASL) - (84.6 * ASW)
+ *
+ * ASL = number of words / number of sentences
+ * ASW = number of syllables / number of words
+ *
+ * The score ranges from 0 (really difficult to read) to 100 (really easy to
+ * read).
+ *
+ * @see https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch_reading_ease
+ */
+#[AutoconfigureTag('deepl.readability')]
+final class FleschKincaidEnglish extends AbstractReadabilityCalculator
+{
+    protected const LANGUAGE = 'en-us';
+    protected const HYPHENATION_LOCALE = 'en_US';
+
+    protected function calculateScore(float $averageSentenceLength, float $averageSyllablesPerWord): float
+    {
+        return 206.835 - 1.015 * $averageSentenceLength - 84.6 * $averageSyllablesPerWord;
+    }
+}

Classes/Readability/Calculator/FleschKincaidFrench.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\DeeplWrite\Readability\Calculator;
+
+use Symfony\Component\DependencyInjection\Attribute\AutoconfigureTag;
+
+/**
+ * Flesch reading-ease score for French, using the Kandel & Moles (1958)
+ * adaptation of the original Flesch formula:
+ *
+ * FRE = 207 - (1.015 * ASL) - (73.6 * ASW)
+ *
+ * ASL = number of words / number of sentences
+ * ASW = number of syllables / number of words
+ *
+ * The score ranges from 0 (really difficult to read) to 100 (really easy to
+ * read).
+ *
+ * @see https://fr.wikipedia.org/wiki/Lisibilit%C3%A9#Formule_de_Kandel_et_Moles
+ * @see https://www.mba-ks.com/analyse-lisibilite-flesch/
+ */
+#[AutoconfigureTag('deepl.readability')]
+final class FleschKincaidFrench extends AbstractReadabilityCalculator
+{
+    protected const LANGUAGE = 'fr';
+    protected const HYPHENATION_LOCALE = 'fr';
+
+    protected function calculateScore(float $averageSentenceLength, float $averageSyllablesPerWord): float
+    {
+        return 207.0 - 1.015 * $averageSentenceLength - 73.6 * $averageSyllablesPerWord;
+    }
+}

Classes/Readability/Calculator/FleschKincaidGerman.php

@@ -0,0 +1,34 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\DeeplWrite\Readability\Calculator;
+
+use Symfony\Component\DependencyInjection\Attribute\AutoconfigureTag;
+
+/**
+ * Flesch reading-ease score for German, using the Amstad (1978) adaptation of
+ * the original Flesch formula:
+ *
+ * FRE = 180 - ASL - (58.5 * ASW)
+ *
+ * ASL = number of words / number of sentences
+ * ASW = number of syllables / number of words
+ *
+ * The score ranges from 0 (really difficult to read) to 100 (really easy to
+ * read).
+ *
+ * @see https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch_reading_ease
+ * @see https://de.wikipedia.org/wiki/Lesbarkeitsindex#F%C3%BCr_Deutsch
+ */
+#[AutoconfigureTag('deepl.readability')]
+final class FleschKincaidGerman extends AbstractReadabilityCalculator
+{
+    protected const LANGUAGE = 'de';
+    protected const HYPHENATION_LOCALE = 'de_DE';
+
+    protected function calculateScore(float $averageSentenceLength, float $averageSyllablesPerWord): float
+    {
+        return 180.0 - $averageSentenceLength - 58.5 * $averageSyllablesPerWord;
+    }
+}