diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3628283..4f4f83d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -53,6 +53,14 @@ jobs: - name: Install tailor run: composer global require typo3/tailor --prefer-dist --no-progress --no-suggest + # Installs the bundled contrib libraries into `contrib/Libraries` so they are + # packaged into the TER artifact for classic mode installations. The committed + # `contrib/composer.lock` is resolved with `--prefer-lowest` (enforced by the + # `checkContribComposer` integration check), so `install` ships the lowest + # supported library versions. + - name: Install contrib libraries + run: composer install -d contrib --no-dev + # Note that step will fail when `env.version` does not match the `ext_emconf.php` version. - name: Create local TER package upload artifact env: diff --git a/.github/workflows/testcore13.yml b/.github/workflows/testcore13.yml index e499b18..04c17d7 100644 --- a/.github/workflows/testcore13.yml +++ b/.github/workflows/testcore13.yml @@ -34,6 +34,9 @@ jobs: - name: "Ensure UTF-8 files do not contain BOM" run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkBom" + - name: "Verify bundled contrib library composer constraint and lock" + run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkContribComposer" + # - name: "Test .rst files for integrity" # run: "Build/Scripts/runTests.sh -t 13 -p ${{ matrix.php-version }} -s checkRst" diff --git a/.gitignore b/.gitignore index 2e706f5..ffc162a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ .Build !.Build/Web/.gitkeep .ddev/docker-compose.tempfs.yaml -composer.lock +/composer.lock composer.json.testing composer.json.orig composer.json.pretty @@ -22,3 +22,4 @@ Build/node_modules/ vendor tailor-version-artefact/ tailor-version-upload/ +/contrib/Libraries/ diff --git a/Build/Scripts/checkContribComposer.sh b/Build/Scripts/checkContribComposer.sh new file mode 100755 index 0000000..cbf9996 --- /dev/null +++ b/Build/Scripts/checkContribComposer.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# +# Integration check for the bundled contrib library shipped in `contrib/Libraries` +# for classic mode / TER installations. +# +# It ensures that: +# 1. the version constraint of the bundled package is identical in the root +# `composer.json` and in `contrib/composer.json`, and +# 2. `contrib/composer.lock` is up to date and resolved with `--prefer-lowest`, +# so the lowest supported library version is the one shipped. +# +# Run via: Build/Scripts/runTests.sh -s checkContribComposer +# + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../" && pwd)" +cd "${ROOT_DIR}" + +PACKAGE="org_heigl/hyphenator" +LOCK="contrib/composer.lock" +LOCK_BACKUP="contrib/composer.lock.checkorig" + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +echo "[1/2] Checking '${PACKAGE}' constraint parity between composer.json and contrib/composer.json ..." +ROOT_CONSTRAINT="$(php -r '$d = json_decode(file_get_contents("composer.json"), true); echo $d["require"][$argv[1]] ?? "";' "${PACKAGE}")" +CONTRIB_CONSTRAINT="$(php -r '$d = json_decode(file_get_contents("contrib/composer.json"), true); echo $d["require"][$argv[1]] ?? "";' "${PACKAGE}")" + +[ -n "${ROOT_CONSTRAINT}" ] || fail "'${PACKAGE}' is not required in composer.json" +[ -n "${CONTRIB_CONSTRAINT}" ] || fail "'${PACKAGE}' is not required in contrib/composer.json" +if [ "${ROOT_CONSTRAINT}" != "${CONTRIB_CONSTRAINT}" ]; then + echo " composer.json: ${ROOT_CONSTRAINT}" >&2 + echo " contrib/composer.json: ${CONTRIB_CONSTRAINT}" >&2 + fail "'${PACKAGE}' constraint differs between composer.json and contrib/composer.json" +fi +echo " OK: both pin '${PACKAGE}' to '${ROOT_CONSTRAINT}'" + +echo "[2/2] Checking ${LOCK} is up to date (--prefer-lowest) ..." +cp "${LOCK}" "${LOCK_BACKUP}" +trap 'mv -f "${LOCK_BACKUP}" "${LOCK}" 2>/dev/null || true' EXIT +composer update --prefer-lowest --no-install --no-interaction --no-progress -d contrib >/dev/null +if ! diff -q "${LOCK}" "${LOCK_BACKUP}" >/dev/null; then + echo "--- committed ${LOCK}" >&2 + echo "+++ freshly resolved (--prefer-lowest)" >&2 + diff "${LOCK_BACKUP}" "${LOCK}" >&2 || true + fail "${LOCK} is out of date. Regenerate it with: composer update --prefer-lowest -d contrib" +fi +echo " OK: ${LOCK} matches contrib/composer.json (--prefer-lowest)" + +echo "SUCCESS" diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh index b395b93..36fdbc2 100755 --- a/Build/Scripts/runTests.sh +++ b/Build/Scripts/runTests.sh @@ -147,6 +147,7 @@ Options: - checkBom: check UTF-8 files do not contain BOM - checkExceptionCodes: Check for duplicate exception codes - checkRst: test .rst files for integrity + - checkContribComposer: verify bundled contrib library constraint parity and prefer-lowest lock freshness - checkTestMethodsPrefix: check tests methods do not start with "test" - clean: clean up build, cache and testing related files and folders - cleanCache: clean up cache related files and folders @@ -472,6 +473,11 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name cgl-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" SUITE_EXIT_CODE=$? ;; + checkContribComposer) + COMMAND="Build/Scripts/checkContribComposer.sh" + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-contrib-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" + SUITE_EXIT_CODE=$? + ;; checkRst) COMMAND="php -dxdebug.mode=off Build/Scripts/validateRstFiles.php" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name cgl-${SUFFIX} -e COMPOSER_CACHE_DIR=.Build/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" diff --git a/Classes/Controller/ReadabilityController.php b/Classes/Controller/ReadabilityController.php new file mode 100644 index 0000000..068bf93 --- /dev/null +++ b/Classes/Controller/ReadabilityController.php @@ -0,0 +1,44 @@ +getParsedBody(); + $data = is_array($data) ? $data : []; + $language = (string)($data['language'] ?? ''); + $text = strip_tags((string)($data['text'] ?? '')); + + try { + $result = $this->factory->fromLanguage($language)->calculateReadability($text); + } catch (\InvalidArgumentException) { + // No readability calculator is registered for the requested language, + // or the text contained no countable words. Report "no score" instead + // of failing the request, so the editor overlay keeps working. + return new JsonResponse(['score' => null]); + } + + return new JsonResponse($result->jsonSerialize()); + } +} diff --git a/Classes/Readability/Calculator/AbstractReadabilityCalculator.php b/Classes/Readability/Calculator/AbstractReadabilityCalculator.php new file mode 100644 index 0000000..b633d15 --- /dev/null +++ b/Classes/Readability/Calculator/AbstractReadabilityCalculator.php @@ -0,0 +1,118 @@ +countSentences($text); + $words = $this->countWords($text); + $syllables = $this->countSyllables($text); + $characters = $this->countCharacters($text); + + if ($words <= 0) { + throw new \InvalidArgumentException( + 'Readability can not be calculated for a text without countable words.', + 1783350001 + ); + } + if ($sentences <= 0) { + $sentences = 1; + } + + // Very short or very simple texts can push the formula above 100. As + // 100 is the defined maximum ("very easy to read") the score is capped + // here; this is a known property of the Flesch formulas and acceptable + // for the quick overview this feature provides. + $score = min( + $this->calculateScore($words / $sentences, $syllables / $words), + 100.0, + ); + + return new ReadabilityResult($text, $sentences, $words, $syllables, $characters, $score); + } + + /** + * Calculate the language specific Flesch reading-ease score. + * + * @param float $averageSentenceLength number of words / number of sentences (ASL) + * @param float $averageSyllablesPerWord number of syllables / number of words (ASW) + */ + abstract protected function calculateScore(float $averageSentenceLength, float $averageSyllablesPerWord): float; + + final protected function countSentences(string $text): int + { + $sentences = preg_split(self::SENTENCE_SPLIT, $text); + if ($sentences === false) { + return 0; + } + return count($sentences); + } + + protected function countWords(string $text): int + { + // Count words as sequences of unicode letters, keeping intra-word + // apostrophes and hyphens together (e.g. "don't", "arc-en-ciel"). + // Unlike str_word_count() this handles the accented characters used by + // languages such as French, Italian, Portuguese, Spanish and German. + return (int)preg_match_all('/\p{L}+(?:[\x27\x{2019}-]\p{L}+)*/u', $text); + } + + final protected function countSyllables(string $text): int + { + $hyphenator = new Hyphenator(); + $options = $hyphenator->getOptions(); + $options->setDefaultLocale(static::HYPHENATION_LOCALE); + $options->setHyphen('|'); + $result = $hyphenator->hyphenate($text); + if (!is_string($result)) { + return 0; + } + $splitted = preg_split(self::HYPHENATED_SPLIT, $result); + if ($splitted === false) { + return 0; + } + return count($splitted); + } + + protected function countCharacters(string $text): int + { + return mb_strlen($text); + } + + public function getLanguage(): string + { + return static::LANGUAGE; + } +} diff --git a/Classes/Readability/Calculator/FleschKincaidEnglish.php b/Classes/Readability/Calculator/FleschKincaidEnglish.php new file mode 100644 index 0000000..43618f5 --- /dev/null +++ b/Classes/Readability/Calculator/FleschKincaidEnglish.php @@ -0,0 +1,33 @@ +registry->findByLanguage($language); + } +} diff --git a/Classes/Readability/ReadabilityCalculatorInterface.php b/Classes/Readability/ReadabilityCalculatorInterface.php new file mode 100644 index 0000000..e7e7b90 --- /dev/null +++ b/Classes/Readability/ReadabilityCalculatorInterface.php @@ -0,0 +1,13 @@ + + */ + private array $services = []; + + /** + * @param iterable $calculators + */ + public function __construct( + #[AutowireIterator('deepl.readability')] + iterable $calculators, + ) { + foreach ($calculators as $calculator) { + $this->services[] = $calculator; + } + } + + public function findByLanguage(string $language): ReadabilityCalculatorInterface + { + $normalized = $this->normalizeLanguage($language); + foreach ($this->services as $service) { + if ($this->normalizeLanguage($service->getLanguage()) === $normalized) { + return $service; + } + } + throw new \InvalidArgumentException( + sprintf('No readability calculator found for language "%s"', $language), + 1757686580 + ); + } + + /** + * Reduce a language tag to its lowercased primary subtag, so that regional + * variants such as "en-US", "en-GB" or "en_us" all resolve to the same + * calculator (here: English), matching the two-letter locale codes that + * CKEditor reports for the editing context. + */ + private function normalizeLanguage(string $language): string + { + $language = str_replace('_', '-', strtolower(trim($language))); + return explode('-', $language)[0]; + } +} diff --git a/Classes/Readability/ReadabilityCalculatorRegistryInterface.php b/Classes/Readability/ReadabilityCalculatorRegistryInterface.php new file mode 100644 index 0000000..99a69b2 --- /dev/null +++ b/Classes/Readability/ReadabilityCalculatorRegistryInterface.php @@ -0,0 +1,10 @@ +words/$this->sentences, 2); + } + + public function getAverageSyllablesPerWord(): float + { + return round($this->syllables/$this->words, 2); + } + + /** + * @return array{ + * sentences: int, + * words: int, + * syllables: int, + * characters: int, + * avgSyllables: float, + * avgWords: float, + * score: float, + * } + */ + public function jsonSerialize(): array + { + return [ + 'sentences' => $this->sentences, + 'words' => $this->words, + 'syllables' => $this->syllables, + 'characters' => $this->characters, + 'avgSyllables' => $this->getAverageSyllablesPerWord(), + 'avgWords' => $this->getAverageWordsPerSentence(), + 'score' => $this->score, + ]; + } +} diff --git a/Configuration/Backend/AjaxRoutes.php b/Configuration/Backend/AjaxRoutes.php index ac9ca05..80bd92c 100644 --- a/Configuration/Backend/AjaxRoutes.php +++ b/Configuration/Backend/AjaxRoutes.php @@ -1,6 +1,7 @@ [ @@ -18,4 +19,9 @@ 'target' => CkEditorController::class . '::getEditMaskAction', 'methods' => ['GET'], ], + 'deeplwrite_readability' => [ + 'path' => '/deepl-write/readability/calculate', + 'target' => ReadabilityController::class . '::calculate', + 'methods' => ['POST'], + ], ]; diff --git a/Documentation/Changelog/2.0/Feature-AddedReadabilityScore.rst b/Documentation/Changelog/2.0/Feature-AddedReadabilityScore.rst new file mode 100644 index 0000000..53c3e05 --- /dev/null +++ b/Documentation/Changelog/2.0/Feature-AddedReadabilityScore.rst @@ -0,0 +1,111 @@ +.. _feature-1783343929: + +========================================= +Feature: Readability score in RTE overlay +========================================= + +Description +=========== + +The :guilabel:`DeepL Write` RTE overlay now displays a Flesch reading-ease +score for both the original and the optimized text, giving editors immediate +feedback on how readable a text is before and after rephrasing. + +The score is shown as a coloured marker bar above each text area and is +updated for the optimized text once the rephrasing result is returned. + +Readability is calculated per content language using a language specific +Flesch reading-ease formula. German, English, French, Italian and Portuguese +are supported out of the box, and regional variants such as ``en-US`` or +``pt-BR`` are matched by their primary language subtag. For unsupported +languages or empty texts no score is reported and the overlay keeps working +without interruption. + +.. figure:: /Images/Editor/deeplwrite-rte-optimize-with-readability-score.png + :alt: DeepL Write RTE overlay showing the readability score for the original and the optimized text + +Impact +====== + +Editors using the :guilabel:`DeepL Write` overlay in the rich text editor +get an at-a-glance readability indication for the text they are working on, +helping to judge whether a rephrasing actually improves readability. + +Developer +========= + +.. seealso:: + + This how-to is also available as dedicated documentation under + :ref:`Developer › Readability calculators `. + +The readability score is computed by small, language specific calculators. Each +calculator implements ``ReadabilityCalculatorInterface`` (usually by extending +``AbstractReadabilityCalculator``) and is registered through the +``deepl.readability`` service tag. At runtime ``ReadabilityCalculatorRegistry`` +selects the calculator whose language matches the primary subtag of the editor +content language. + +Out of the box the following languages are provided, each using a documented +Flesch reading-ease adaptation, where ``ASL`` is the average sentence length +(words / sentences) and ``ASW`` the average number of syllables per word +(syllables / words): + +* English (``en``) — Flesch: ``206.835 - 1.015 * ASL - 84.6 * ASW`` +* German (``de``) — Amstad: ``180 - ASL - 58.5 * ASW`` +* French (``fr``) — Kandel & Moles: ``207 - 1.015 * ASL - 73.6 * ASW`` +* Italian (``it``) — Franchina & Vacca: ``206 - ASL - 65 * ASW`` +* Portuguese (``pt``) — Martins et al.: ``248.835 - 1.015 * ASL - 84.6 * ASW`` + +Adding a language +----------------- + +To support an additional language, add a class that extends +``AbstractReadabilityCalculator``, declare the language it serves and the +hyphenation locale used for syllable counting, and implement the language +specific formula. Tag it with ``#[AutoconfigureTag('deepl.readability')]`` so it +is registered automatically: + +.. code-block:: php + + ` + + Learn how the readability score is calculated and how to add custom + readability calculators for additional languages. + +.. toctree:: + :maxdepth: 2 + :titlesonly: + :hidden: + + ReadabilityCalculators/Index diff --git a/Documentation/Developer/ReadabilityCalculators/Index.rst b/Documentation/Developer/ReadabilityCalculators/Index.rst new file mode 100644 index 0000000..f49e047 --- /dev/null +++ b/Documentation/Developer/ReadabilityCalculators/Index.rst @@ -0,0 +1,106 @@ +.. _developer-readability-calculators: + +======================= +Readability Calculators +======================= + +The :guilabel:`DeepL Write` overlay shows a Flesch reading-ease score for the +edited text (see :ref:`the feature description `). The score +is computed by small, language specific *readability calculators* that can be +extended by third-party extensions. + +How it works +============ + +Each calculator implements +``WebVision\DeeplWrite\Readability\ReadabilityCalculatorInterface`` – usually by +extending ``WebVision\DeeplWrite\Readability\Calculator\AbstractReadabilityCalculator`` +– and is registered through the ``deepl.readability`` service tag. At runtime +``ReadabilityCalculatorRegistry`` selects the calculator whose language matches +the primary subtag of the editor content language, so ``en-US`` and ``en-GB`` +both resolve to the English calculator. If no calculator matches, or the text +contains no countable words, no score is reported and the overlay keeps working. + +The base class provides the shared text metrics – counting sentences, words +(unicode aware) and syllables (via ``org_heigl/hyphenator``) – guards against +empty input and caps the score at the maximum of 100. A concrete calculator +therefore only has to provide the language specific formula. + +Supported languages +=================== + +The following languages ship with a documented Flesch reading-ease adaptation, +where ``ASL`` is the average sentence length (words / sentences) and ``ASW`` the +average number of syllables per word (syllables / words): + +.. list-table:: + :header-rows: 1 + + * - Language + - Formula + - Adaptation + * - English (``en``) + - ``206.835 - 1.015 * ASL - 84.6 * ASW`` + - Flesch + * - German (``de``) + - ``180 - ASL - 58.5 * ASW`` + - Amstad + * - French (``fr``) + - ``207 - 1.015 * ASL - 73.6 * ASW`` + - Kandel & Moles + * - Italian (``it``) + - ``206 - ASL - 65 * ASW`` + - Franchina & Vacca + * - Portuguese (``pt``) + - ``248.835 - 1.015 * ASL - 84.6 * ASW`` + - Martins et al. + +Adding a language +================= + +To support an additional language, add a class that extends +``AbstractReadabilityCalculator``, declare the language it serves (``LANGUAGE``) +and the hyphenation locale used for syllable counting (``HYPHENATION_LOCALE``, +matching a dictionary shipped by ``org_heigl/hyphenator`` such as ``de``, +``en``, ``es``, ``fr``, ``it`` or ``pt``), and implement the language specific +formula in ``calculateScore()``. Tag it with +``#[AutoconfigureTag('deepl.readability')]`` so it is picked up automatically: + +.. code-block:: php + + ` + + Extend the extension, for example by adding readability calculators + for additional languages. + .. card:: :ref:`Changelog ` Learn about what have changed and what actions are required to process. @@ -77,6 +82,7 @@ localizing records. Administration/Index Reference/Index KnownIssues/Index + Developer/Index Changelog/Index .. Meta Menu diff --git a/README.md b/README.md index 14180e6..44a1937 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,28 @@ configured for both extension in their respective extension configuration. > api key is required for this extension, which can also be used for the > `deepltranslate-core` or using there a free key. +### RTE Styling implementation + +A styling implementation is added to improve your writing directly inside your +RTE. This needs some adjustments to your RTE configuration: + +```yaml +editor: + config: + importModules: + # your already existing modules + - { module: '@web-vision/deepl-write/deeplwrite-plugin.js', exports: [ 'Deeplwrite' ] } + toolbar: + items: + # Your existing configuration + - '|' + - deeplwrite +``` + +This adds a button to the RTE controls, which allows you to use the overlay for +the writing style. The button is shown disabled, if your API key is not allowed +using the DeepL Write API or no API key is set. + ## Sponsors We appreciate very much the sponsorships of the developments and features in diff --git a/Resources/Private/Core13/Backend/Templates/CkEditor/Edit.html b/Resources/Private/Core13/Backend/Templates/CkEditor/Edit.html index 367031f..c4066fb 100644 --- a/Resources/Private/Core13/Backend/Templates/CkEditor/Edit.html +++ b/Resources/Private/Core13/Backend/Templates/CkEditor/Edit.html @@ -4,6 +4,67 @@ xmlns:deeplWrite="http://typo3.org/ns/WebVision/DeeplWrite/ViewHelpers" data-namespace-typo3-fluid="true" > +

@@ -88,12 +149,24 @@