From f5bdc1ac24c84f8b936d7375ce57224127b35369 Mon Sep 17 00:00:00 2001 From: Aryan Sharma Date: Thu, 1 Oct 2026 23:44:45 +0530 Subject: [PATCH 1/7] feat: add `@stdlib/math/base/special/gelu` Signed-off-by: Aryan Sharma --- .../@stdlib/math/base/special/gelu/README.md | 125 ++++++++++++++++ .../base/special/gelu/benchmark/benchmark.js | 52 +++++++ .../math/base/special/gelu/docs/repl.txt | 30 ++++ .../base/special/gelu/docs/types/index.d.ts | 48 ++++++ .../math/base/special/gelu/docs/types/test.ts | 44 ++++++ .../math/base/special/gelu/examples/index.js | 30 ++++ .../math/base/special/gelu/lib/index.js | 49 +++++++ .../math/base/special/gelu/lib/main.js | 66 +++++++++ .../math/base/special/gelu/package.json | 137 ++++++++++++++++++ .../math/base/special/gelu/test/test.js | 91 ++++++++++++ 10 files changed, 672 insertions(+) create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/README.md create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/benchmark/benchmark.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/docs/repl.txt create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/docs/types/index.d.ts create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/docs/types/test.ts create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/examples/index.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/lib/index.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/lib/main.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/package.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/test.js diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/README.md b/lib/node_modules/@stdlib/math/base/special/gelu/README.md new file mode 100644 index 000000000000..60e96b4c96a3 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/README.md @@ -0,0 +1,125 @@ + + +# gelu + +> Evaluate the [Gaussian error linear unit][gelu] (GELU). + +
+ +The [Gaussian error linear unit][gelu] (GELU) activation function is defined as: + +```math +\mathrm{gelu}(x) = x \cdot \Phi(x) = \frac{x}{2} \left[ 1 + \mathrm{erf}\left( \frac{x}{\sqrt{2}} \right) \right] +``` + +where $\Phi(x)$ is the cumulative distribution function of the standard normal distribution and $\mathrm{erf}(x)$ is the [error function][@stdlib/math/base/special/erf]. + +
+ + + +
+ +## Usage + +```javascript +var gelu = require( '@stdlib/math/base/special/gelu' ); +``` + +#### gelu( x ) + +Evaluates the Gaussian Error Linear Unit (GELU) activation function. + +```javascript +var v = gelu( 0.0 ); +// returns 0.0 + +v = gelu( 1.0 ); +// returns ~0.8413 + +v = gelu( -Infinity ); +// returns 0.0 + +v = gelu( Infinity ); +// returns Infinity + +v = gelu( NaN ); +// returns NaN +``` + +
+ + + +
+ +## Examples + + + +```javascript +var logEachMap = require( '@stdlib/console/log-each-map' ); +var uniform = require( '@stdlib/random/array/uniform' ); +var gelu = require( '@stdlib/math/base/special/gelu' ); + +var opts = { + 'dtype': 'float64' +}; +var x = uniform( 100, -5.0, 5.0, opts ); + +logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); +``` + +
+ + + + + + + + + + diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/benchmark/benchmark.js b/lib/node_modules/@stdlib/math/base/special/gelu/benchmark/benchmark.js new file mode 100644 index 000000000000..19e6b947c103 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/benchmark/benchmark.js @@ -0,0 +1,52 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +// MODULES // + +var bench = require( '@stdlib/bench' ); +var uniform = require( '@stdlib/random/array/uniform' ); +var isnan = require( '@stdlib/math/base/assert/is-nan' ); +var pkg = require( './../package.json' ).name; +var gelu = require( './../lib' ); + + +// MAIN // + +bench( pkg, function benchmark( b ) { + var x; + var y; + var i; + + x = uniform( 100, -10.0, 10.0 ); + + b.tic(); + for ( i = 0; i < b.iterations; i++ ) { + y = gelu( x[ i % x.length ] ); + if ( isnan( y ) ) { + b.fail( 'should not return NaN' ); + } + } + b.toc(); + if ( isnan( y ) ) { + b.fail( 'should not return NaN' ); + } + b.pass( 'benchmark finished' ); + b.end(); +}); diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/docs/repl.txt b/lib/node_modules/@stdlib/math/base/special/gelu/docs/repl.txt new file mode 100644 index 000000000000..6ab1f30c27ef --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/docs/repl.txt @@ -0,0 +1,30 @@ + +{{alias}}( x ) + Evaluates the Gaussian Error Linear Unit (GELU) activation function. + + Parameters + ---------- + x: number + Input value. + + Returns + ------- + y: number + Function value. + + Examples + -------- + > var y = {{alias}}( 0.0 ) + 0.0 + > y = {{alias}}( 1.0 ) + ~0.8413 + > y = {{alias}}( -Infinity ) + 0.0 + > y = {{alias}}( Infinity ) + Infinity + > y = {{alias}}( NaN ) + NaN + + See Also + -------- + diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/index.d.ts b/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/index.d.ts new file mode 100644 index 000000000000..025866ff00e8 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/index.d.ts @@ -0,0 +1,48 @@ +/* +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +// TypeScript Version: 4.1 + +/** +* Evaluate the Gaussian Error Linear Unit (GELU) activation function. +* +* @param x - input value +* @returns function value +* +* @example +* var y = gelu( 0.0 ); +* // returns 0.0 +* +* @example +* var y = gelu( 1.0 ); +* // returns ~0.8413 +* +* @example +* var y = gelu( -Infinity ); +* // returns 0.0 +* +* @example +* var y = gelu( NaN ); +* // returns NaN +*/ +declare function gelu( x: number ): number; + + +// EXPORTS // + +export = gelu; diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/test.ts b/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/test.ts new file mode 100644 index 000000000000..d6b3ad284b88 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/docs/types/test.ts @@ -0,0 +1,44 @@ +/* +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import gelu = require( './index' ); + + +// TESTS // + +// The function returns a number... +{ + gelu( 8 ); // $ExpectType number +} + +// The compiler throws an error if the function is provided a value other than a number... +{ + gelu( true ); // $ExpectError + gelu( false ); // $ExpectError + gelu( null ); // $ExpectError + gelu( undefined ); // $ExpectError + gelu( '5' ); // $ExpectError + gelu( [] ); // $ExpectError + gelu( {} ); // $ExpectError + gelu( ( x: number ): number => x ); // $ExpectError +} + +// The compiler throws an error if the function is provided insufficient arguments... +{ + gelu(); // $ExpectError +} diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/examples/index.js b/lib/node_modules/@stdlib/math/base/special/gelu/examples/index.js new file mode 100644 index 000000000000..dd2d87fa8fb7 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/examples/index.js @@ -0,0 +1,30 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +var logEachMap = require( '@stdlib/console/log-each-map' ); +var uniform = require( '@stdlib/random/array/uniform' ); +var gelu = require( './../lib' ); + +var opts = { + 'dtype': 'float64' +}; +var x = uniform( 100, -5.0, 5.0, opts ); + +logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/lib/index.js b/lib/node_modules/@stdlib/math/base/special/gelu/lib/index.js new file mode 100644 index 000000000000..070dfeb3691d --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/lib/index.js @@ -0,0 +1,49 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +/** +* Evaluate the Gaussian Error Linear Unit (GELU) activation function. +* +* @module @stdlib/math/base/special/gelu +* +* @example +* var gelu = require( '@stdlib/math/base/special/gelu' ); +* +* var y = gelu( 0.0 ); +* // returns 0.0 +* +* y = gelu( 1.0 ); +* // returns ~0.8413 +* +* y = gelu( -Infinity ); +* // returns 0.0 +* +* y = gelu( NaN ); +* // returns NaN +*/ + +// MODULES // + +var main = require( './main.js' ); + + +// EXPORTS // + +module.exports = main; diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/lib/main.js b/lib/node_modules/@stdlib/math/base/special/gelu/lib/main.js new file mode 100644 index 000000000000..288f535d3b2e --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/lib/main.js @@ -0,0 +1,66 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +// MODULES // + +var SQRT_HALF = require( '@stdlib/constants/float64/sqrt-half' ); +var isnan = require( '@stdlib/math/base/assert/is-nan' ); +var erfc = require( '@stdlib/math/base/special/erfc' ); +var NINF = require( '@stdlib/constants/float64/ninf' ); + + +// MAIN // + +/** +* Evaluates the Gaussian Error Linear Unit (GELU) activation function. +* +* @param {number} x - input value +* @returns {number} evaluated GELU +* +* @example +* var v = gelu( 0.0 ); +* // returns 0.0 +* +* @example +* var v = gelu( 1.0 ); +* // returns ~0.8413 +* +* @example +* var v = gelu( -Infinity ); +* // returns 0.0 +* +* @example +* var v = gelu( NaN ); +* // returns NaN +*/ +function gelu( x ) { + if ( isnan( x ) ) { + return NaN; + } + if ( x === NINF ) { + return 0.0; + } + return 0.5 * x * erfc( -x * SQRT_HALF ); +} + + +// EXPORTS // + +module.exports = gelu; diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/package.json b/lib/node_modules/@stdlib/math/base/special/gelu/package.json new file mode 100644 index 000000000000..13de3b6b3247 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/package.json @@ -0,0 +1,137 @@ +{ + "name": "@stdlib/math/base/special/gelu", + "version": "0.0.0", + "description": "Evaluate the Gaussian Error Linear Unit (GELU).", + "license": "Apache-2.0", + "author": { + "name": "The Stdlib Authors", + "url": "https://github.com/stdlib-js/stdlib/graphs/contributors" + }, + "contributors": [ + { + "name": "The Stdlib Authors", + "url": "https://github.com/stdlib-js/stdlib/graphs/contributors" + } + ], + "main": "./lib", + "directories": { + "benchmark": "./benchmark", + "doc": "./docs", + "example": "./examples", + "lib": "./lib", + "test": "./test" + }, + "types": "./docs/types", + "scripts": {}, + "homepage": "https://github.com/stdlib-js/stdlib", + "repository": { + "type": "git", + "url": "git://github.com/stdlib-js/stdlib.git" + }, + "bugs": { + "url": "https://github.com/stdlib-js/stdlib/issues" + }, + "dependencies": {}, + "devDependencies": {}, + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "keywords": [ + "stdlib", + "stdmath", + "mathematics", + "math", + "gelu", + "activation", + "gaussian", + "neural", + "deep learning" + ], + "__stdlib__": { + "scaffold": { + "$schema": "math/base@v1.0", + "base_alias": "gelu", + "alias": "gelu", + "pkg_desc": "evaluate the Gaussian Error Linear Unit (GELU) for a double-precision floating-point number", + "desc": "evaluates the Gaussian Error Linear Unit (GELU) for a double-precision floating-point number", + "short_desc": "Gaussian error linear unit", + "parameters": [ + { + "name": "x", + "desc": "input value", + "type": { + "javascript": "number", + "jsdoc": "number", + "c": "double", + "dtype": "float64" + }, + "domain": [ + { + "min": "-infinity", + "max": "infinity" + } + ], + "rand": { + "prng": "random/base/uniform", + "parameters": [ + -5, + 5 + ] + }, + "example_values": [ + -4.62, + 3.17, + -2.41, + 1.28, + -0.73, + 2.56, + -3.09, + 0.41, + 4.12, + -1.85, + 2.93, + -4.37, + 1.67, + -0.28, + 3.74, + -2.96, + 0.83, + -1.12, + 4.89, + -3.51 + ] + } + ], + "output_policy": "real_floating_point_and_generic", + "returns": { + "desc": "function value", + "type": { + "javascript": "number", + "jsdoc": "number", + "c": "double", + "dtype": "float64" + } + }, + "keywords": [ + "gelu", + "activation", + "gaussian", + "neural", + "deep learning" + ], + "extra_keywords": [] + } + } +} diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/test/test.js b/lib/node_modules/@stdlib/math/base/special/gelu/test/test.js new file mode 100644 index 000000000000..db880776ac31 --- /dev/null +++ b/lib/node_modules/@stdlib/math/base/special/gelu/test/test.js @@ -0,0 +1,91 @@ +/** +* @license Apache-2.0 +* +* Copyright (c) 2026 The Stdlib Authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +// MODULES // + +var tape = require( 'tape' ); +var isAlmostSameValue = require( '@stdlib/assert/is-almost-same-value' ); +var isnan = require( '@stdlib/math/base/assert/is-nan' ); +var PINF = require( '@stdlib/constants/float64/pinf' ); +var NINF = require( '@stdlib/constants/float64/ninf' ); +var gelu = require( './../lib' ); + + +// TESTS // + +tape( 'main export is a function', function test( t ) { + t.ok( true, __filename ); + t.strictEqual( typeof gelu, 'function', 'main export is a function' ); + t.end(); +}); + +tape( 'the function returns `NaN` when provided `NaN`', function test( t ) { + var y = gelu( NaN ); + t.strictEqual( isnan( y ), true, 'returns expected value' ); + t.end(); +}); + +tape( 'the function returns `0.0` when provided `0.0`', function test( t ) { + var y = gelu( 0.0 ); + t.strictEqual( y, 0.0, 'returns expected value' ); + t.end(); +}); + +tape( 'the function returns `+Infinity` when provided `+Infinity`', function test( t ) { + var y = gelu( PINF ); + t.strictEqual( y, PINF, 'returns expected value' ); + t.end(); +}); + +tape( 'the function returns `0.0` when provided `-Infinity`', function test( t ) { + var y = gelu( NINF ); + t.strictEqual( y, 0.0, 'returns expected value' ); + t.end(); +}); + +tape( 'the function evaluates the GELU function for representative inputs', function test( t ) { + var expected; + var actual; + var values; + var i; + + values = [ + 1.0, + -1.0, + 2.0, + -2.0, + 5.0, + -5.0 + ]; + expected = [ + 0.8413447460685429, + -0.15865525393145705, + 1.9544997361036415, + -0.04550026389635842, + 4.999998566742141, + -0.0000014332578593959682 + ]; + + for ( i = 0; i < values.length; i++ ) { + actual = gelu( values[ i ] ); + t.strictEqual( isAlmostSameValue( actual, expected[ i ], 2 ), true, 'returns expected value for ' + values[ i ] ); + } + t.end(); +}); From 890f6860e37a98e302e5b850c1b01dbf2ff8f912 Mon Sep 17 00:00:00 2001 From: Aryan Sharma Date: Thu, 8 Oct 2026 23:55:22 +0530 Subject: [PATCH 2/7] feat: add tanh and quick approximations to math/base/special/gelu Signed-off-by: Aryan Sharma --- .../@stdlib/math/base/special/gelu/README.md | 75 ++++++- .../base/special/gelu/benchmark/benchmark.js | 44 +++++ .../math/base/special/gelu/docs/repl.txt | 59 +++++- .../base/special/gelu/docs/types/index.d.ts | 89 ++++++++- .../math/base/special/gelu/docs/types/test.ts | 22 +++ .../math/base/special/gelu/examples/index.js | 2 + .../math/base/special/gelu/lib/index.js | 29 ++- .../math/base/special/gelu/lib/main.js | 12 +- .../math/base/special/gelu/lib/quick.js | 69 +++++++ .../math/base/special/gelu/lib/tanh.js | 81 ++++++++ .../math/base/special/gelu/package.json | 12 +- .../fixtures/python/gelu_large_negative.json | 1 + .../fixtures/python/gelu_large_positive.json | 1 + .../test/fixtures/python/gelu_medium.json | 1 + .../test/fixtures/python/gelu_near_zero.json | 1 + .../test/fixtures/python/gelu_special.json | 1 + .../test/fixtures/python/gelu_subnormal.json | 1 + .../fixtures/python/quick_large_negative.json | 1 + .../fixtures/python/quick_large_positive.json | 1 + .../test/fixtures/python/quick_medium.json | 1 + .../test/fixtures/python/quick_near_zero.json | 1 + .../test/fixtures/python/quick_special.json | 1 + .../gelu/test/fixtures/python/runner.py | 157 +++++++++++++++ .../fixtures/python/tanh_large_negative.json | 1 + .../fixtures/python/tanh_large_positive.json | 1 + .../test/fixtures/python/tanh_medium.json | 1 + .../test/fixtures/python/tanh_near_zero.json | 1 + .../test/fixtures/python/tanh_special.json | 1 + .../math/base/special/gelu/test/test.js | 59 +----- .../math/base/special/gelu/test/test.main.js | 187 ++++++++++++++++++ .../math/base/special/gelu/test/test.quick.js | 171 ++++++++++++++++ .../math/base/special/gelu/test/test.tanh.js | 171 ++++++++++++++++ 32 files changed, 1188 insertions(+), 67 deletions(-) create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/lib/quick.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/lib/tanh.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_large_negative.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_large_positive.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_medium.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_near_zero.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_special.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/gelu_subnormal.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/quick_large_negative.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/quick_large_positive.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/quick_medium.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/quick_near_zero.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/quick_special.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/runner.py create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/tanh_large_negative.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/tanh_large_positive.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/tanh_medium.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/tanh_near_zero.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/fixtures/python/tanh_special.json create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/test.main.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/test.quick.js create mode 100644 lib/node_modules/@stdlib/math/base/special/gelu/test/test.tanh.js diff --git a/lib/node_modules/@stdlib/math/base/special/gelu/README.md b/lib/node_modules/@stdlib/math/base/special/gelu/README.md index 60e96b4c96a3..90ac1f8b90a3 100644 --- a/lib/node_modules/@stdlib/math/base/special/gelu/README.md +++ b/lib/node_modules/@stdlib/math/base/special/gelu/README.md @@ -26,11 +26,15 @@ limitations under the License. The [Gaussian error linear unit][gelu] (GELU) activation function is defined as: + + ```math -\mathrm{gelu}(x) = x \cdot \Phi(x) = \frac{x}{2} \left[ 1 + \mathrm{erf}\left( \frac{x}{\sqrt{2}} \right) \right] +\mathrm{gelu}(x) = x \cdot \Phi(x) = \frac{x}{2} \, \mathrm{erfc}\left( -\frac{x}{\sqrt{2}} \right) ``` -where $\Phi(x)$ is the cumulative distribution function of the standard normal distribution and $\mathrm{erf}(x)$ is the [error function][@stdlib/math/base/special/erf]. + + +where $\Phi(x)$ is the cumulative distribution function of the standard normal distribution and $\mathrm{erfc}(x)$ is the [complementary error function][@stdlib/math/base/special/erfc]. @@ -56,7 +60,7 @@ v = gelu( 1.0 ); // returns ~0.8413 v = gelu( -Infinity ); -// returns 0.0 +// returns -0.0 v = gelu( Infinity ); // returns Infinity @@ -65,6 +69,64 @@ v = gelu( NaN ); // returns NaN ``` +#### gelu.tanh( x ) + +Evaluates the Gaussian Error Linear Unit (GELU) activation function using a tanh approximation: + + + +```math +\mathrm{gelu}(x) \approx \frac{x}{2} \left( 1 + \tanh\left( \sqrt{\frac{2}{\pi}} \left( x + 0.044715 \, x^3 \right) \right) \right) +``` + + + +```javascript +var v = gelu.tanh( 0.0 ); +// returns 0.0 + +v = gelu.tanh( 1.0 ); +// returns ~0.8412 + +v = gelu.tanh( -Infinity ); +// returns -0.0 + +v = gelu.tanh( Infinity ); +// returns Infinity + +v = gelu.tanh( NaN ); +// returns NaN +``` + +#### gelu.quick( x ) + +Evaluates the Gaussian Error Linear Unit (GELU) activation function using a quick 1.702 approximation: + + + +```math +\mathrm{gelu}(x) \approx x \cdot \operatorname{expit}( 1.702 \, x ) +``` + + + +```javascript +var v = gelu.quick( 0.0 ); +// returns 0.0 + +v = gelu.quick( 1.0 ); +// returns ~0.8458 + +v = gelu.quick( -Infinity ); +// returns -0.0 + +v = gelu.quick( Infinity ); +// returns Infinity + +v = gelu.quick( NaN ); +// returns NaN +``` + @@ -86,6 +148,8 @@ var opts = { var x = uniform( 100, -5.0, 5.0, opts ); logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); +logEachMap( 'gelu.tanh(%0.4f) = %0.4f', x, gelu.tanh ); +logEachMap( 'gelu.quick(%0.4f) = %0.4f', x, gelu.quick ); ``` @@ -100,6 +164,7 @@ logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); - [`@stdlib/math/base/special/erf`][@stdlib/math/base/special/erf]: evaluate the error function. - [`@stdlib/math/base/special/erfc`][@stdlib/math/base/special/erfc]: evaluate the complementary error function. +- [`@stdlib/math/base/special/expit`][@stdlib/math/base/special/expit]: evaluate the standard logistic function. - [`@stdlib/math/base/special/ramp`][@stdlib/math/base/special/ramp]: evaluate the ramp function. @@ -108,7 +173,7 @@ logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); -Evaluates the Gaussian Error Linear Unit (GELU) activation function using a tanh approximation: + - +
-```math -\mathrm{gelu}(x) \approx \frac{x}{2} \left( 1 + \tanh\left( \sqrt{\frac{2}{\pi}} \left( x + 0.044715 \, x^3 \right) \right) \right) -``` +## Examples - + ```javascript -var v = gelu.tanh( 0.0 ); -// returns 0.0 +var uniform = require( '@stdlib/random/array/uniform' ); +var logEachMap = require( '@stdlib/console/log-each-map' ); +var gelu = require( '@stdlib/math/base/special/gelu' ); -v = gelu.tanh( 1.0 ); -// returns ~0.8412 +var opts = { + 'dtype': 'float64' +}; +var x = uniform( 100, -5.0, 5.0, opts ); -v = gelu.tanh( -Infinity ); -// returns -0.0 +logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); +``` -v = gelu.tanh( Infinity ); -// returns Infinity +
-v = gelu.tanh( NaN ); -// returns NaN -``` + + + -#### gelu.quick( x ) +* * * -Evaluates the Gaussian Error Linear Unit (GELU) activation function using a quick 1.702 approximation: +
- +## C APIs -```math -\mathrm{gelu}(x) \approx x \cdot \operatorname{expit}( 1.702 \, x ) + + +
+ +
+ + + + + +
+ +### Usage + +```c +#include "stdlib/math/base/special/gelu.h" ``` - +#### stdlib_base_gelu( x ) -```javascript -var v = gelu.quick( 0.0 ); -// returns 0.0 +Evaluates the Gaussian Error Linear Unit (GELU) activation function. -v = gelu.quick( 1.0 ); -// returns ~0.8458 +```c +double y = stdlib_base_gelu( 1.0 ); +// returns ~0.8413 +``` -v = gelu.quick( -Infinity ); -// returns -0.0 +The function accepts the following arguments: -v = gelu.quick( Infinity ); -// returns Infinity +- **x**: `[in] double` input value. -v = gelu.quick( NaN ); -// returns NaN +```c +double stdlib_base_gelu( const double x ); ```
-
+ -## Examples +
- +
-```javascript -var logEachMap = require( '@stdlib/console/log-each-map' ); -var uniform = require( '@stdlib/random/array/uniform' ); -var gelu = require( '@stdlib/math/base/special/gelu' ); + -var opts = { - 'dtype': 'float64' -}; -var x = uniform( 100, -5.0, 5.0, opts ); + -logEachMap( 'gelu(%0.4f) = %0.4f', x, gelu ); -logEachMap( 'gelu.tanh(%0.4f) = %0.4f', x, gelu.tanh ); -logEachMap( 'gelu.quick(%0.4f) = %0.4f', x, gelu.quick ); +
+ +### Examples + +```c +#include "stdlib/math/base/special/gelu.h" +#include + +int main( void ) { + const double x[] = { -5.0, -3.89, -2.78, -1.67, -0.56, 0.56, 1.67, 2.78, 3.89, 5.0 }; + + double v; + int i; + for ( i = 0; i < 10; i++ ) { + v = stdlib_base_gelu( x[ i ] ); + printf( "gelu(%lf) = %lf\n", x[ i ], v ); + } +} ```
+
+ + + + + @@ -173,7 +199,7 @@ logEachMap( 'gelu.quick(%0.4f) = %0.4f', x, gelu.quick );