diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..10fe8399 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Format with Runic +da0580609466b256075c58cb1257e94769a46419 diff --git a/.github/workflows/Format.yml b/.github/workflows/Format.yml new file mode 100644 index 00000000..77873d1c --- /dev/null +++ b/.github/workflows/Format.yml @@ -0,0 +1,31 @@ +name: Format + +on: + pull_request: + branches: + - master + push: + branches: + - master + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + runic: + name: Runic + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + - uses: julia-actions/cache@v3 + - uses: fredrikekre/runic-action@v1 + with: + version: '1' + extensions: 'jl,md' + docstrings: 'true' diff --git a/README.md b/README.md index 29c0f08b..3775bddf 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliadiff.org/ReverseDiff.jl/dev) [![Build status](https://github.com/JuliaDiff/ReverseDiff.jl/workflows/CI/badge.svg)](https://github.com/JuliaDiff/ReverseDiff.jl/actions) [![codecov.io](https://codecov.io/github/JuliaDiff/ReverseDiff.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaDiff/ReverseDiff.jl?branch=master) +[![code style: runic](https://img.shields.io/badge/code_style-%E1%9A%B1%E1%9A%A2%E1%9A%BE%E1%9B%81%E1%9A%B2-black)](https://github.com/fredrikekre/Runic.jl) ReverseDiff is a fast and compile-able tape-based **reverse mode automatic differentiation (AD)** that implements methods to take **gradients**, **Jacobians**, **Hessians**, and @@ -21,7 +22,7 @@ differentiation methods (such as [finite differencing](https://en.wikipedia.org/ To install ReverseDiff, simply use Julia's package manager: -```julia +```julia-repl julia> Pkg.add("ReverseDiff") ``` diff --git a/benchmark/benchmark.jl b/benchmark/benchmark.jl index aee08875..ff5bdd01 100644 --- a/benchmark/benchmark.jl +++ b/benchmark/benchmark.jl @@ -53,11 +53,12 @@ function grad_benchmark_driver!(out, f, x) else println("skipping compiled GradientTape benchmark because the tape is too long ($(length(tp.tape)) elements)") end + return nothing end #################################################################### -rosenbrock(x) = sum(map(ReverseDiff.@forward((i, j) -> (1 - j)^2 + 100*(i - j^2)^2), x[2:end], x[1:end-1])) +rosenbrock(x) = sum(map(ReverseDiff.@forward((i, j) -> (1 - j)^2 + 100 * (i - j^2)^2), x[2:end], x[1:(end - 1)])) # function rosenbrock(x) # i = x[2:end] @@ -82,16 +83,18 @@ grad_benchmark_driver!(out, rosenbrock, x) #################################################################### function ackley(x::AbstractVector) - a, b, c = 20.0, -0.2, 2.0*π + a, b, c = 20.0, -0.2, 2.0 * π len_recip = inv(length(x)) sum_sqrs = zero(eltype(x)) sum_cos = sum_sqrs for i in x - sum_cos += cos(c*i) + sum_cos += cos(c * i) sum_sqrs += i^2 end - return (-a * exp(b * sqrt(len_recip*sum_sqrs)) - - exp(len_recip*sum_cos) + a + e) + return ( + -a * exp(b * sqrt(len_recip * sum_sqrs)) - + exp(len_recip * sum_cos) + a + e + ) end x = rand(100000) @@ -103,8 +106,8 @@ grad_benchmark_driver!(out, ackley, x) function generate_matrix_test(n) return x -> begin @assert length(x) == 2n^2 + n - a = reshape(x[1:n^2], n, n) - b = reshape(x[n^2 + 1:2n^2], n, n) + a = reshape(x[1:(n^2)], n, n) + b = reshape(x[(n^2 + 1):2n^2], n, n) return trace(log.((a * b) + a - b)) end end @@ -119,7 +122,7 @@ grad_benchmark_driver!(out, matrix_test, x) relu(x) = log.(1.0 .+ exp.(x)) -ReverseDiff.@forward sigmoid(n) = 1. / (1. + exp(-n)) +ReverseDiff.@forward sigmoid(n) = 1.0 / (1.0 + exp(-n)) function neural_net(w1, w2, w3, x1) x2 = relu(w1 * x1) @@ -127,7 +130,7 @@ function neural_net(w1, w2, w3, x1) return sigmoid(dot(w3, x3)) end -xs = (randn(10,10), randn(10,10), randn(10), rand(10)) +xs = (randn(10, 10), randn(10, 10), randn(10), rand(10)) outs = map(similar, xs) grad_benchmark_driver!(outs, neural_net, xs) diff --git a/docs/make.jl b/docs/make.jl index 41617830..11a0a48e 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,19 +1,19 @@ using Documenter, ReverseDiff makedocs(; - modules=[ReverseDiff], - sitename="ReverseDiff.jl", - format=Documenter.HTML(; - prettyurls=get(ENV, "CI", nothing) == "true", + modules = [ReverseDiff], + sitename = "ReverseDiff.jl", + format = Documenter.HTML(; + prettyurls = get(ENV, "CI", nothing) == "true", ), - pages=[ + pages = [ "Home" => "index.md", "Limitation of ReverseDiff" => "limits.md", "API" => "api.md", ], - checkdocs=:exports, + checkdocs = :exports, ) deploydocs(; - repo="github.com/JuliaDiff/ReverseDiff.jl.git", push_preview=true, + repo = "github.com/JuliaDiff/ReverseDiff.jl.git", push_preview = true, ) diff --git a/ext/StatisticsExt.jl b/ext/StatisticsExt.jl index d856448d..b6a7cd59 100644 --- a/ext/StatisticsExt.jl +++ b/ext/StatisticsExt.jl @@ -1,10 +1,10 @@ module StatisticsExt using ReverseDiff: ReverseDiff, SpecialInstruction, TrackedArray, deriv, increment_deriv!, - istracked, record!, tape, track, unseed!, value, value! + istracked, record!, tape, track, unseed!, value, value! using Statistics: Statistics, mean -function Statistics.mean(x::TrackedArray{V,D}) where {V,D} +function Statistics.mean(x::TrackedArray{V, D}) where {V, D} tp = tape(x) out = track(mean(value(x)), D, tp) record!(tp, SpecialInstruction, mean, x, out) diff --git a/src/ReverseDiff.jl b/src/ReverseDiff.jl index 011a026f..575f747c 100644 --- a/src/ReverseDiff.jl +++ b/src/ReverseDiff.jl @@ -25,12 +25,12 @@ using ChainRulesCore const ARRAY_TYPES = (:AbstractArray, :AbstractVector, :AbstractMatrix, :Array, :Vector, :Matrix) const REAL_TYPES = (:Bool, :Integer, :AbstractIrrational, :Rational, :BigFloat, :BigInt, :AbstractFloat, :Real, :Dual) -const SKIPPED_UNARY_SCALAR_FUNCS = Symbol[:isinf, :isnan, :isfinite, :iseven, :isodd, :isreal, :isinteger, :exponent] +const SKIPPED_UNARY_SCALAR_FUNCS = Symbol[:isinf, :isnan, :isfinite, :iseven, :isodd, :isreal, :isinteger, :exponent] const SKIPPED_BINARY_SCALAR_FUNCS = Symbol[:isequal, :isless, :<, :>, :(==), :(!=), :(<=), :(>=)] # Some functions with derivatives in DiffRules are not supported # For instance, ReverseDiff does not support functions with complex results and derivatives -const SKIPPED_DIFFRULES = Tuple{Symbol,Symbol}[ +const SKIPPED_DIFFRULES = Tuple{Symbol, Symbol}[ (:SpecialFunctions, :hankelh1), (:SpecialFunctions, :hankelh1x), (:SpecialFunctions, :hankelh2), diff --git a/src/api/Config.jl b/src/api/Config.jl index 3876b51a..d28198b9 100644 --- a/src/api/Config.jl +++ b/src/api/Config.jl @@ -42,11 +42,11 @@ GradientConfig(input::Tuple, tp::InstructionTape = InstructionTape()) = Gradient Like `GradientConfig(input, tp)`, except the provided type `D` is assumed to be the element type of the target function's output. """ -function GradientConfig(input::Tuple, ::Type{D}, tp::InstructionTape = InstructionTape()) where D +function GradientConfig(input::Tuple, ::Type{D}, tp::InstructionTape = InstructionTape()) where {D} return _GradientConfig(map(x -> track(similar(x), D, tp), input), tp) end -function GradientConfig(input::AbstractArray, ::Type{D}, tp::InstructionTape = InstructionTape()) where D +function GradientConfig(input::AbstractArray, ::Type{D}, tp::InstructionTape = InstructionTape()) where {D} return _GradientConfig(track(similar(input), D, tp), tp) end @@ -54,16 +54,16 @@ end # JacobianConfig # ################## -struct JacobianConfig{I,O} <: AbstractConfig +struct JacobianConfig{I, O} <: AbstractConfig input::I output::O tape::InstructionTape # disable default outer constructor - JacobianConfig{I,O}(input, output, tape) where {I,O} = new{I,O}(input, output, tape) + JacobianConfig{I, O}(input, output, tape) where {I, O} = new{I, O}(input, output, tape) end # "private" convienence constructor -_JacobianConfig(input::I, output::O, tape::InstructionTape) where {I,O} = JacobianConfig{I,O}(input, output, tape) +_JacobianConfig(input::I, output::O, tape::InstructionTape) where {I, O} = JacobianConfig{I, O}(input, output, tape) """ ReverseDiff.JacobianConfig(input, tp::InstructionTape = InstructionTape()) @@ -99,16 +99,16 @@ stored or modified in any way. See `ReverseDiff.jacobian` for a description of acceptable types for `input`. """ -function JacobianConfig(output::AbstractArray{D}, input::Tuple, tp::InstructionTape = InstructionTape()) where D +function JacobianConfig(output::AbstractArray{D}, input::Tuple, tp::InstructionTape = InstructionTape()) where {D} cfg_input = map(x -> track(similar(x), D, tp), input) - cfg_output = track!(similar(output, TrackedReal{D,D,Nothing}), output, tp) + cfg_output = track!(similar(output, TrackedReal{D, D, Nothing}), output, tp) return _JacobianConfig(cfg_input, cfg_output, tp) end # we dispatch on V<:Real here because InstructionTape is actually also an AbstractArray -function JacobianConfig(output::AbstractArray{D}, input::AbstractArray{V}, tp::InstructionTape = InstructionTape()) where {D,V<:Real} +function JacobianConfig(output::AbstractArray{D}, input::AbstractArray{V}, tp::InstructionTape = InstructionTape()) where {D, V <: Real} cfg_input = track(similar(input), D, tp) - cfg_output = track!(similar(output, TrackedReal{D,D,Nothing}), output, tp) + cfg_output = track!(similar(output, TrackedReal{D, D, Nothing}), output, tp) return _JacobianConfig(cfg_input, cfg_output, tp) end @@ -123,7 +123,7 @@ JacobianConfig(result::DiffResult, input, tp::InstructionTape) = JacobianConfig( # HessianConfig # ################# -struct HessianConfig{G<:GradientConfig,J<:JacobianConfig} <: AbstractConfig +struct HessianConfig{G <: GradientConfig, J <: JacobianConfig} <: AbstractConfig gradient_config::G jacobian_config::J end @@ -149,7 +149,7 @@ end Like `HessianConfig(input, tp)`, except the provided type `D` is assumed to be the element type of the target function's output. """ -function HessianConfig(input::AbstractArray, ::Type{D}, gtp::InstructionTape = InstructionTape(), jtp::InstructionTape = InstructionTape()) where D +function HessianConfig(input::AbstractArray, ::Type{D}, gtp::InstructionTape = InstructionTape(), jtp::InstructionTape = InstructionTape()) where {D} jcfg = JacobianConfig(input, D, jtp) gcfg = GradientConfig(jcfg.input, gtp) return HessianConfig(gcfg, jcfg) diff --git a/src/api/gradients.jl b/src/api/gradients.jl index 234a44a8..b52801fa 100644 --- a/src/api/gradients.jl +++ b/src/api/gradients.jl @@ -50,7 +50,7 @@ end ########################### """ - ReverseDiff.gradient!(tape::Union{GradientTape,CompiledGradient}, input) + ReverseDiff.gradient!(tape::Union{GradientTape, CompiledGradient}, input) If `input` is an `AbstractArray`, assume `tape` represents a function of the form `f(::AbstractArray)::Real` and return `∇f(input)`. @@ -59,14 +59,14 @@ If `input` is a tuple of `AbstractArray`s, assume `tape` represents a function o `f(::AbstractArray...)::Real` and return a `Tuple` where the `i`th element is the gradient of `f` w.r.t. `input[i].` """ -function gradient!(tape::Union{GradientTape,CompiledGradient}, input) +function gradient!(tape::Union{GradientTape, CompiledGradient}, input) result = construct_result(input_hook(tape)) result = gradient!(result, tape, input) return result end """ - ReverseDiff.gradient!(result, tape::Union{GradientTape,CompiledGradient}, input) + ReverseDiff.gradient!(result, tape::Union{GradientTape, CompiledGradient}, input) This method is exactly like `ReverseDiff.gradient!(tape, input)`, except it stores the resulting gradient(s) in `result` rather than allocating new memory. @@ -77,7 +77,7 @@ case the primal value `f(input)` (or `f(input...)`, if `isa(input, Tuple)`) will in it as well. An immutable `DiffResult` cannot be updated in place and is replaced, so use the returned value: `result = ReverseDiff.gradient!(result, tape, input)`. """ -function gradient!(result, tape::Union{GradientTape,CompiledGradient}, input) +function gradient!(result, tape::Union{GradientTape, CompiledGradient}, input) seeded_forward_pass!(tape, input) result = seeded_reverse_pass!(result, tape) return result diff --git a/src/api/hessians.jl b/src/api/hessians.jl index db66fec2..cd02f725 100644 --- a/src/api/hessians.jl +++ b/src/api/hessians.jl @@ -44,17 +44,21 @@ function hessian!(result, f, input::AbstractArray, cfg::HessianConfig = HessianC return result end -function hessian!(result::DiffResult, f, input::AbstractArray, - cfg::HessianConfig = HessianConfig(result, input)) +function hessian!( + result::DiffResult, f, input::AbstractArray, + cfg::HessianConfig = HessianConfig(result, input) + ) ∇f! = (y, x) -> begin gradient_result = DiffResult(zero(eltype(y)), y) gradient_result = gradient!(gradient_result, f, x, cfg.gradient_config) result = DiffResults.value!(result, value(DiffResults.value(gradient_result))) return y end - jacobian!(DiffResults.hessian(result), ∇f!, - DiffResults.gradient(result), input, - cfg.jacobian_config) + jacobian!( + DiffResults.hessian(result), ∇f!, + DiffResults.gradient(result), input, + cfg.jacobian_config + ) return result end @@ -63,21 +67,21 @@ end ########################## """ - ReverseDiff.hessian!(tape::Union{HessianTape,CompiledHessian}, input) + ReverseDiff.hessian!(tape::Union{HessianTape, CompiledHessian}, input) Assuming `tape` represents a function of the form `f(::AbstractArray{<:Real})::Real`, return the Hessian `H(f)(input)`. """ -function hessian!(tape::Union{HessianTape,CompiledHessian}, input::AbstractArray) +function hessian!(tape::Union{HessianTape, CompiledHessian}, input::AbstractArray) result = construct_result(output_hook(tape), input_hook(tape)) result = hessian!(result, tape, input) return result end """ - ReverseDiff.hessian!(result::AbstractArray, tape::Union{HessianTape,CompiledHessian}, input) + ReverseDiff.hessian!(result::AbstractArray, tape::Union{HessianTape, CompiledHessian}, input) - ReverseDiff.hessian!(result::DiffResult, tape::Union{HessianTape,CompiledHessian}, input) + ReverseDiff.hessian!(result::DiffResult, tape::Union{HessianTape, CompiledHessian}, input) This method is exactly like `ReverseDiff.hessian!(tape, input)`, except it stores the resulting Hessian in `result` rather than allocating new memory. @@ -87,13 +91,13 @@ If `result` is a `DiffResults.DiffResult`, the primal value `f(input)` and the g `DiffResult` cannot be updated in place and is replaced, so use the returned value: `result = ReverseDiff.hessian!(result, tape, input)`. """ -function hessian!(result::AbstractArray, tape::Union{HessianTape,CompiledHessian}, input::AbstractArray) +function hessian!(result::AbstractArray, tape::Union{HessianTape, CompiledHessian}, input::AbstractArray) seeded_forward_pass!(tape, input) result = seeded_reverse_pass!(result, tape) return result end -function hessian!(result::DiffResult, tape::Union{HessianTape,CompiledHessian}, input::AbstractArray) +function hessian!(result::DiffResult, tape::Union{HessianTape, CompiledHessian}, input::AbstractArray) seeded_forward_pass!(tape, input) inner = DiffResult(DiffResults.gradient(result), DiffResults.hessian(result)) inner = seeded_reverse_pass!(inner, tape) diff --git a/src/api/jacobians.jl b/src/api/jacobians.jl index 58c9fc5a..925d0bdd 100644 --- a/src/api/jacobians.jl +++ b/src/api/jacobians.jl @@ -86,7 +86,7 @@ end ########################### """ - ReverseDiff.jacobian!(tape::Union{JacobianTape,CompiledJacobian}, input) + ReverseDiff.jacobian!(tape::Union{JacobianTape, CompiledJacobian}, input) If `input` is an `AbstractArray`, assume `tape` represents a function of the form `f(::AbstractArray{<:Real})::AbstractArray{<:Real}` or `f!(::AbstractArray{<:Real}, @@ -102,14 +102,14 @@ execute `tape` with new `input` values. There is no way to re-run `tape`'s tape `output` values; since `f!` can mutate `output`, there exists no stable "hook" for loading new `output` values into the tape. """ -function jacobian!(tape::Union{JacobianTape,CompiledJacobian}, input) +function jacobian!(tape::Union{JacobianTape, CompiledJacobian}, input) result = construct_result(output_hook(tape), input_hook(tape)) result = jacobian!(result, tape, input) return result end """ - ReverseDiff.jacobian!(result, tape::Union{JacobianTape,CompiledJacobian}, input) + ReverseDiff.jacobian!(result, tape::Union{JacobianTape, CompiledJacobian}, input) This method is exactly like `ReverseDiff.jacobian!(tape, input)`, except it stores the resulting Jacobian(s) in `result` rather than allocating new memory. @@ -120,7 +120,7 @@ case the primal value of the target function will be stored in it as well. An im `DiffResult` cannot be updated in place and is replaced, so use the returned value: `result = ReverseDiff.jacobian!(result, tape, input)`. """ -function jacobian!(result, tape::Union{JacobianTape,CompiledJacobian}, input) +function jacobian!(result, tape::Union{JacobianTape, CompiledJacobian}, input) seeded_forward_pass!(tape, input) result = seeded_reverse_pass!(result, tape) return result diff --git a/src/api/tape.jl b/src/api/tape.jl index 1be1f973..d3e290b8 100644 --- a/src/api/tape.jl +++ b/src/api/tape.jl @@ -11,17 +11,17 @@ Base.show(io::IO, t::AbstractTape) = print(io, typeof(t).name, "(", t.func, ")") for T in (:GradientTape, :JacobianTape, :HessianTape) _T = Symbol(string("_", T)) @eval begin - struct $(T){F,I,O} <: AbstractTape + struct $(T){F, I, O} <: AbstractTape func::F input::I output::O tape::InstructionTape # disable default outer constructor - $(T){F,I,O}(func, input, output, tape) where {F,I,O} = new{F,I,O}(func, input, output, tape) + $(T){F, I, O}(func, input, output, tape) where {F, I, O} = new{F, I, O}(func, input, output, tape) end # "private" convienence constructor - $(_T)(func::F, input::I, output::O, tape::InstructionTape) where {F,I,O} = $(T){F,I,O}(func, input, output, tape) + $(_T)(func::F, input::I, output::O, tape::InstructionTape) where {F, I, O} = $(T){F, I, O}(func, input, output, tape) Base.length(t::$T) = length(t.tape) @@ -52,7 +52,7 @@ end # CompiledTape # ################ -struct CompiledTape{T<:AbstractTape} <: AbstractTape +struct CompiledTape{T <: AbstractTape} <: AbstractTape tape::T forward_exec::Vector{FunctionWrapper{Nothing, Tuple{}}} reverse_exec::Vector{FunctionWrapper{Nothing, Tuple{}}} @@ -93,23 +93,24 @@ end @inline (e::ReverseExecutor)() = reverse_exec!(e.instruction) """ - (::Type{CompiledTape}){T<:AbstractTape}(t::T) + (::Type{CompiledTape}){T <: AbstractTape}(t::T) Construct a compiled type by wrapping the `forward_exec!` and `reverse_exec!` methods on each instruction in the tape. """ -function CompiledTape(t::T) where T<:AbstractTape - CompiledTape{T}(t, +function CompiledTape(t::T) where {T <: AbstractTape} + return CompiledTape{T}( + t, [FunctionWrapper{Nothing, Tuple{}}(ForwardExecutor(instruction)) for instruction in t.tape], [FunctionWrapper{Nothing, Tuple{}}(ReverseExecutor(t.tape[i])) for i in length(t.tape):-1:1] - ) + ) end Base.show(io::IO, t::CompiledTape) = print(io, typeof(t).name, "($(t.tape.func))") -const CompiledGradient{T<:GradientTape} = CompiledTape{T} -const CompiledJacobian{T<:JacobianTape} = CompiledTape{T} -const CompiledHessian{T<:HessianTape} = CompiledTape{T} +const CompiledGradient{T <: GradientTape} = CompiledTape{T} +const CompiledJacobian{T <: JacobianTape} = CompiledTape{T} +const CompiledHessian{T <: HessianTape} = CompiledTape{T} Base.length(ct::CompiledTape) = length(ct.tape) @@ -123,14 +124,14 @@ function forward_pass!(compiled_tape::CompiledTape) for wrapper in compiled_tape.forward_exec wrapper() end - nothing + return nothing end function reverse_pass!(compiled_tape::CompiledTape) for wrapper in compiled_tape.reverse_exec wrapper() end - nothing + return nothing end """ @@ -149,31 +150,37 @@ function compile(t::AbstractTape) end function compile_gradient(f, args...) - Base.depwarn("`ReverseDiff.compile_gradient(f, args...)` is deprecated" * - ", use `ReverseDiff.compile(ReverseDiff.GradientTape(f, args...))`"* - "instead. Then, you can execute the returned CompiledTape `t` by calling"* - " `ReverseDiff.gradient!(result, t, input)`.", - :compile_gradient) + Base.depwarn( + "`ReverseDiff.compile_gradient(f, args...)` is deprecated" * + ", use `ReverseDiff.compile(ReverseDiff.GradientTape(f, args...))`" * + "instead. Then, you can execute the returned CompiledTape `t` by calling" * + " `ReverseDiff.gradient!(result, t, input)`.", + :compile_gradient + ) tape = compile(GradientTape(f, args...)) return (result, input) -> gradient!(result, tape, input) end function compile_jacobian(f, args...) - Base.depwarn("`ReverseDiff.compile_jacobian(f, args...)` is deprecated" * - ", use `ReverseDiff.compile(ReverseDiff.JacobianTape(f, args...))`"* - "instead. Then, you can execute the returned CompiledTape `t` by calling"* - " `ReverseDiff.jacobian!(result, t, input)`.", - :compile_jacobian) + Base.depwarn( + "`ReverseDiff.compile_jacobian(f, args...)` is deprecated" * + ", use `ReverseDiff.compile(ReverseDiff.JacobianTape(f, args...))`" * + "instead. Then, you can execute the returned CompiledTape `t` by calling" * + " `ReverseDiff.jacobian!(result, t, input)`.", + :compile_jacobian + ) tape = compile(JacobianTape(f, args...)) return (result, input) -> jacobian!(result, tape, input) end function compile_hessian(f, args...) - Base.depwarn("`ReverseDiff.compile_hessian(f, args...)` is deprecated" * - ", use `ReverseDiff.compile(ReverseDiff.HessianTape(f, args...))`"* - "instead. Then, you can execute the returned CompiledTape `t` by calling"* - " `ReverseDiff.hessian!(result, t, input)`.", - :compile_hessian) + Base.depwarn( + "`ReverseDiff.compile_hessian(f, args...)` is deprecated" * + ", use `ReverseDiff.compile(ReverseDiff.HessianTape(f, args...))`" * + "instead. Then, you can execute the returned CompiledTape `t` by calling" * + " `ReverseDiff.hessian!(result, t, input)`.", + :compile_hessian + ) tape = compile(HessianTape(f, args...)) return (result, input) -> hessian!(result, tape, input) end diff --git a/src/api/utils.jl b/src/api/utils.jl index 147cbd80..bb6fb33c 100644 --- a/src/api/utils.jl +++ b/src/api/utils.jl @@ -63,7 +63,7 @@ function seeded_reverse_pass!(result::DiffResult, output::AbstractArray, input:: return result end -function seeded_reverse_pass!(result::NTuple{N,Any}, output::AbstractArray, input::NTuple{N,Any}, tape) where {N} +function seeded_reverse_pass!(result::NTuple{N, Any}, output::AbstractArray, input::NTuple{N, Any}, tape) where {N} return map((r, i) -> seeded_reverse_pass!(r, output, i, tape), result, input) end @@ -71,7 +71,7 @@ end # result extraction # ##################### -function extract_result!(result::NTuple{N,Any}, output, input::NTuple{N,Any}) where {N} +function extract_result!(result::NTuple{N, Any}, output, input::NTuple{N, Any}) where {N} return map((r, i) -> extract_result!(r, output, i), result, input) end diff --git a/src/derivatives/arrays.jl b/src/derivatives/arrays.jl index 879ae224..b717ff85 100644 --- a/src/derivatives/arrays.jl +++ b/src/derivatives/arrays.jl @@ -3,24 +3,25 @@ ########## function Base.fill( - value::TrackedReal, - dims::Vararg{Union{Integer, AbstractUnitRange}}, -) + value::TrackedReal, + dims::Vararg{Union{Integer, AbstractUnitRange}}, + ) return track(fill, value, dims...) end @grad function fill(v::Real, dims...) - return fill(value(v), dims...), function(Δ) - size(Δ) ≢ dims && error("Dimension mismatch") - return (sum(Δ), map(_->nothing, dims)...) + function back(Δ) + size(Δ) ≢ dims && error("Dimension mismatch") + return (sum(Δ), map(_ -> nothing, dims)...) end + return fill(value(v), dims...), back end ############### ## any & all ## ############### -Base.any(f::Function, x::TrackedArray; dims=:) = any(f, value(x), dims = dims) -Base.all(f::Function, x::TrackedArray; dims=:) = all(f, value(x), dims = dims) +Base.any(f::Function, x::TrackedArray; dims = :) = any(f, value(x), dims = dims) +Base.all(f::Function, x::TrackedArray; dims = :) = all(f, value(x), dims = dims) ######### ## cat ## @@ -28,12 +29,12 @@ Base.all(f::Function, x::TrackedArray; dims=:) = all(f, value(x), dims = dims) function combinations(xs, n) n < 1 && return [[]] - cs = combinations(xs, n-1) - [[x, c...] for x in xs, c in cs] + cs = combinations(xs, n - 1) + return [[x, c...] for x in xs, c in cs] end for f in [:hcat, :vcat] - for i = 0:2, c = combinations([:AbstractVector, :Vector, :TrackedVector, :AbstractMatrix, :TrackedMatrix, :Number, :TrackedReal], i) + for i in 0:2, c in combinations([:AbstractVector, :Vector, :TrackedVector, :AbstractMatrix, :TrackedMatrix, :Number, :TrackedReal], i) cnames = map(_ -> gensym(), c) @eval begin Base.$f($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::TrackedVector) = track($f, $(cnames...), x) @@ -41,13 +42,13 @@ for f in [:hcat, :vcat] Base.$f($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::TrackedReal) = track($f, $(cnames...), x) end for T in [ - :AbstractVector, - :AbstractMatrix, - :Number, - :AbstractVecOrMat, - :(Union{AbstractVector, Number}), - :Vector, - ] + :AbstractVector, + :AbstractMatrix, + :Number, + :AbstractVecOrMat, + :(Union{AbstractVector, Number}), + :Vector, + ] @eval begin Base.$f($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::TrackedVector, xs::$T...) = track($f, $(cnames...), x, xs...) Base.$f($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::TrackedMatrix, xs::$T...) = track($f, $(cnames...), x, xs...) @@ -59,14 +60,14 @@ end @grad function vcat(xs::Union{Number, AbstractVecOrMat}...) xs_value = value.(xs) - out_value = vcat(xs_value...,) + out_value = vcat(xs_value...) function back(Δ) start = 0 Δs = map(xs) do xsi if xsi isa Number - d = Δ[start+1] + d = Δ[start + 1] else - d = Δ[start+1:start+size(xsi,1), :] + d = Δ[(start + 1):(start + size(xsi, 1)), :] end start += size(xsi, 1) d @@ -78,47 +79,48 @@ end @grad function hcat(xs::Union{Number, AbstractVecOrMat}...) xs_value = value.(xs) - out_value = hcat(xs_value...,) + out_value = hcat(xs_value...) function back(Δ) start = 0 Δs = map(xs) do xsi d = if ndims(xsi) == 0 - Δ[start+1] + Δ[start + 1] elseif ndims(xsi) == 1 - Δ[:, start+1] + Δ[:, start + 1] else i = map(_ -> :, size(xsi)) |> Base.tail |> Base.tail - Δ[:, start+1:start+size(xsi,2), i...] + Δ[:, (start + 1):(start + size(xsi, 2)), i...] end start += size(xsi, 2) d end return (Δs...,) - end + end return out_value, back end -for i = 0:2, c = combinations([:AbstractArray, :TrackedArray, :Number, :TrackedReal], i) +for i in 0:2, c in combinations([:AbstractArray, :TrackedArray, :Number, :TrackedReal], i) cnames = map(_ -> gensym(), c) - @eval Base.cat($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::Union{TrackedArray,TrackedReal}, xs::Union{AbstractArray,Number}...; dims) = track(cat, $(cnames...), x, xs...; dims=dims) + @eval Base.cat($([:($x::$c) for (x, c) in zip(cnames, c)]...), x::Union{TrackedArray, TrackedReal}, xs::Union{AbstractArray, Number}...; dims) = track(cat, $(cnames...), x, xs...; dims = dims) end @grad function cat(Xs::Union{Number, AbstractArray}...; dims) Xs_value = value.(Xs) - return cat(Xs_value...; dims = dims), Δ -> begin + function back(Δ) start = ntuple(i -> 0, Val(ndims(Δ))) Δs = map(Xs) do xs if xs isa Number - d = Δ[start+1] + d = Δ[start + 1] start = start .+ 1 else dim_xs = 1:ndims(xs) - till_xs = ntuple((i -> i in dims ? (i in dim_xs ? size(xs,i) : 1) : 0), Val(ndims(Δ))) - xs_in_Δ = ntuple(i -> till_xs[i] > 0 ? (start[i]+1:start[i]+till_xs[i]) : Colon(), Val(ndims(Δ))) - d = reshape(Δ[xs_in_Δ...],size(xs)) + till_xs = ntuple((i -> i in dims ? (i in dim_xs ? size(xs, i) : 1) : 0), Val(ndims(Δ))) + xs_in_Δ = ntuple(i -> till_xs[i] > 0 ? ((start[i] + 1):(start[i] + till_xs[i])) : Colon(), Val(ndims(Δ))) + d = reshape(Δ[xs_in_Δ...], size(xs)) start = start .+ till_xs end d end return (Δs...,) end + return cat(Xs_value...; dims = dims), back end diff --git a/src/derivatives/broadcast.jl b/src/derivatives/broadcast.jl index abecc67f..60d9d38f 100644 --- a/src/derivatives/broadcast.jl +++ b/src/derivatives/broadcast.jl @@ -53,7 +53,7 @@ recur_value(xs) = xs recur_value(xs::Union{TrackedReal, TrackedArray}) = recur_value(value(xs)) function broadcast_rebuild(bc::Broadcasted) - broadcasted(bc.f, broadcast_rebuild.(bc.args)...) + return broadcasted(bc.f, broadcast_rebuild.(bc.args)...) end getstyle(::Broadcasted{Style}) where {Style} = Style @@ -81,14 +81,14 @@ function get_implementation(bc, f, T, args) # No tracked closure or arguments, except TrackedReal and TrackedArray. if !mayhavetracked(bc) && outputisreal && (anyreals(args) || !onlyrealarrays(args)) return Val(:tracker) - # No arg is a real number and array args must be arrays of untracked reals or tracked arrays of reals, - # Output is real, and - # No tracked closure or arguments, except TrackedReal and TrackedArray. elseif !mayhavetracked(bc) && outputisreal + # No arg is a real number and array args must be arrays of untracked reals or tracked arrays of reals, + # Output is real, and + # No tracked closure or arguments, except TrackedReal and TrackedArray. return Val(:reversediff) - # Function or any arg is possibly a tracked non-real or an array of tracked reals/non-reals, - # Or output is not an array of reals else + # Function or any arg is possibly a tracked non-real or an array of tracked reals/non-reals, + # Or output is not an array of reals return Val(:fallback) end end @@ -165,8 +165,8 @@ end out_value = DiffResults.value.(results) eltype(out_value) == Bool && return out_value out = track(out_value, D, tp) - cache = (results, df, index_bound.(targs, (out,))) - record!(tp, SpecialInstruction, ∇broadcast, targs, out, cache) + cache = (results, df, index_bound.(targs, (out,))) + record!(tp, SpecialInstruction, ∇broadcast, targs, out, cache) return out end @noinline function special_reverse_exec!(instruction::SpecialInstruction{typeof(∇broadcast)}) @@ -198,7 +198,7 @@ end return Expr(:block, [:(_br_add_to_deriv!(xs[$i], o, r, Val($i), bounds[$i])) for i in 1:N]...) end _br_add_to_deriv!(_, _, _, _, _) = nothing -function _br_add_to_deriv!(x::Union{TrackedReal,TrackedArray}, out_deriv, results, ::Val{i}, bound) where {i} +function _br_add_to_deriv!(x::Union{TrackedReal, TrackedArray}, out_deriv, results, ::Val{i}, bound) where {i} return istracked(x) && diffresult_increment_deriv!(x, out_deriv, results, i, bound) end @@ -218,9 +218,9 @@ end trim(x, Δ) = reshape(Δ, ntuple(i -> size(Δ, i), Val(ndims(x)))) unbroadcast(x::AbstractArray, Δ) = - size(x) == size(Δ) ? Δ : - length(x) == length(Δ) ? trim(x, Δ) : - trim(x, sum(Δ, dims = ntuple(i -> size(x, i) == 1 ? i : ndims(Δ)+1, Val(ndims(Δ))))) + size(x) == size(Δ) ? Δ : + length(x) == length(Δ) ? trim(x, Δ) : + trim(x, sum(Δ, dims = ntuple(i -> size(x, i) == 1 ? i : ndims(Δ) + 1, Val(ndims(Δ))))) unbroadcast(x::Number, Δ) = sum(Δ) unbroadcast(x::Base.RefValue, _) = nothing @@ -229,7 +229,7 @@ dual(x, p) = x dual(x::Real, p) = Dual(x, p) function _deriv(f, G, ::Val{i}, args::Vararg{Any, N}) where {N, i} - dargs = ntuple(j -> dual(args[j], i==j), Val(N)) + dargs = ntuple(j -> dual(args[j], i == j), Val(N)) return f(dargs...).partials[1] * G end @generated function _derivs(f, G, args::Vararg{Any, N}) where {N} @@ -240,9 +240,9 @@ end out_value = broadcast(f, args_values...) tp = tape(args...) eltype(out_value) == Bool && return out_value - out = track(out_value, tp) + out = track(out_value, tp) cache = (f,) - record!(tp, SpecialInstruction, tracker_∇broadcast, args, out, cache) + record!(tp, SpecialInstruction, tracker_∇broadcast, args, out, cache) return out end @@ -273,7 +273,7 @@ end @inline _materialize(f, args) = broadcast(f, args...) -for (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) +for (M, f, arity) in DiffRules.diffrules(; filter_modules = nothing) if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f)) @warn "$M.$f is not available and hence rule for it can not be defined" continue # Skip rules for methods not defined in the current scope @@ -282,22 +282,22 @@ for (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) @eval @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray}}) = _materialize(bc.f, bc.args) elseif arity == 2 @eval begin - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray,TrackedArray}}) = _materialize(bc.f, bc.args) - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray,TrackedReal}}) = _materialize(bc.f, bc.args) - @noinline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedReal,TrackedArray}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray, TrackedArray}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray, TrackedReal}}) = _materialize(bc.f, bc.args) + @noinline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedReal, TrackedArray}}) = _materialize(bc.f, bc.args) end for A in ARRAY_TYPES @eval begin - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{$A{<:Number},TrackedArray}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{$A{<:Number}, TrackedArray}}) = _materialize(bc.f, bc.args) @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray, $A{<:Number}}}) = _materialize(bc.f, bc.args) @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{$A{<:Number}, TrackedReal}}) = _materialize(bc.f, bc.args) - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedReal,$A{<:Number}}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedReal, $A{<:Number}}}) = _materialize(bc.f, bc.args) end end for R in REAL_TYPES @eval begin - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{$R,TrackedArray}}) = _materialize(bc.f, bc.args) - @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray,$R}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{$R, TrackedArray}}) = _materialize(bc.f, bc.args) + @inline materialize(bc::RDBroadcasted{typeof($M.$f), <:Tuple{TrackedArray, $R}}) = _materialize(bc.f, bc.args) end end end diff --git a/src/derivatives/elementwise.jl b/src/derivatives/elementwise.jl index 130c7aa3..5caf1274 100644 --- a/src/derivatives/elementwise.jl +++ b/src/derivatives/elementwise.jl @@ -67,7 +67,7 @@ end # dispatch # #----------# -for g! in (:map!, :broadcast!), (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) +for g! in (:map!, :broadcast!), (M, f, arity) in DiffRules.diffrules(; filter_modules = nothing) if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f)) @warn "$M.$f is not available and hence rule for it can not be defined" continue # Skip rules for methods not defined in the current scope @@ -102,7 +102,7 @@ end #--------# for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray{S}, x::TrackedArray{X}) where {F,S,X} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray{S}, x::TrackedArray{X}) where {F, S, X} result = DiffResults.DiffResult(zero(S), zero(S)) df = v -> ForwardDiff.derivative!(result, f.f, v) results = $(g)(df, value(x)) @@ -113,7 +113,7 @@ for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) end for TX in (:TrackedArray, :TrackedReal), TY in (:TrackedArray, :TrackedReal) (TX == TY == :TrackedReal) && continue - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray{S}, x::$(TX){X}, y::$(TY){Y}) where {F,S,X,Y} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray{S}, x::$(TX){X}, y::$(TY){Y}) where {F, S, X, Y} result = DiffResults.GradientResult(SVector(zero(S), zero(S))) df = (vx, vy) -> ForwardDiff.gradient!(result, s -> f.f(s[1], s[2]), SVector(vx, vy)) results = $(g)(df, value(x), value(y)) @@ -124,10 +124,10 @@ for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) end end for A in ARRAY_TYPES - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::TrackedReal{X,D}, y::$A) where {F,X,D} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::TrackedReal{X, D}, y::$A) where {F, X, D} result = DiffResults.DiffResult(zero(X), zero(D)) - df = let result=result - (vx, vy) -> let vy=vy + df = let result = result + (vx, vy) -> let vy = vy ForwardDiff.derivative!(result, s -> f.f(s, vy), vx) end end @@ -137,10 +137,10 @@ for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) record!(tape(x), SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::$A, y::TrackedReal{Y,D}) where {F,Y,D} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::$A, y::TrackedReal{Y, D}) where {F, Y, D} result = DiffResults.DiffResult(zero(Y), zero(D)) - df = let result=result - (vx, vy) -> let vx=vx + df = let result = result + (vx, vy) -> let vx = vx ForwardDiff.derivative!(result, s -> f.f(vx, s), vy) end end @@ -150,9 +150,9 @@ for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) record!(tape(y), SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::TrackedArray{X}, y::$A) where {F,X} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::TrackedArray{X}, y::$A) where {F, X} result = DiffResults.GradientResult(SVector(zero(X))) - df = (vx, vy) -> let vy=vy + df = (vx, vy) -> let vy = vy ForwardDiff.gradient!(result, s -> f.f(s[1], vy), SVector(vx)) end results = $(g)(df, value(x), value(y)) @@ -161,9 +161,9 @@ for (g!, g) in ((:map!, :map), (:broadcast!, :broadcast)) record!(tape(x), SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::$A, y::TrackedArray{Y}) where {F,Y} + @eval function Base.$(g!)(f::ForwardOptimize{F}, out::TrackedArray, x::$A, y::TrackedArray{Y}) where {F, Y} result = DiffResults.GradientResult(SVector(zero(Y))) - df = let vx=vx + df = let vx = vx (vx, vy) -> ForwardDiff.gradient!(result, s -> f.f(vx, s[1]), SVector(vy)) end results = $(g)(df, value(x), value(y)) @@ -189,7 +189,7 @@ end # dispatch # #----------# -for g in (:map, :broadcast), (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) +for g in (:map, :broadcast), (M, f, arity) in DiffRules.diffrules(; filter_modules = nothing) if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f)) @warn "$M.$f is not available and hence rule for it can not be defined" continue # Skip rules for methods not defined in the current scope @@ -228,7 +228,7 @@ end #--------# for g in (:map, :broadcast) - @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedArray{X,D}) where {F,X,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedArray{X, D}) where {F, X, D} T = promote_type(X, D) result = DiffResults.DiffResult(zero(T), zero(T)) df = v -> ForwardDiff.derivative!(result, f.f, v) @@ -240,10 +240,10 @@ for g in (:map, :broadcast) return out end for A in ARRAY_TYPES - @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedReal{X,D}, y::$A) where {F,X,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedReal{X, D}, y::$A) where {F, X, D} result = DiffResults.DiffResult(zero(X), zero(D)) - df = let result=result - (vx, vy) -> let vy=vy + df = let result = result + (vx, vy) -> let vy = vy ForwardDiff.derivative!(result, s -> f.f(s, vy), vx) end end @@ -254,10 +254,10 @@ for g in (:map, :broadcast) record!(tp, SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g)(f::ForwardOptimize{F}, x::$A, y::TrackedReal{Y,D}) where {F,Y,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::$A, y::TrackedReal{Y, D}) where {F, Y, D} result = DiffResults.DiffResult(zero(Y), zero(D)) - df = let result=result - (vx, vy) -> let vx=vx + df = let result = result + (vx, vy) -> let vx = vx ForwardDiff.derivative!(result, s -> f.f(vx, s), vy) end end @@ -268,9 +268,9 @@ for g in (:map, :broadcast) record!(tp, SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedArray{X,D}, y::$A) where {F,X,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::TrackedArray{X, D}, y::$A) where {F, X, D} result = DiffResults.GradientResult(SVector(zero(X))) - df = (vx, vy) -> let vy=vy + df = (vx, vy) -> let vy = vy ForwardDiff.gradient!(result, s -> f.f(s[1], vy), SVector(vx)) end results = $(g)(df, value(x), value(y)) @@ -280,9 +280,9 @@ for g in (:map, :broadcast) record!(tp, SpecialInstruction, $(g), (x, y), out, cache) return out end - @eval function Base.$(g)(f::ForwardOptimize{F}, x::$A, y::TrackedArray{Y,D}) where {F,Y,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::$A, y::TrackedArray{Y, D}) where {F, Y, D} result = DiffResults.GradientResult(SVector(zero(Y))) - df = (vx, vy) -> let vx=vx + df = (vx, vy) -> let vx = vx ForwardDiff.gradient!(result, s -> f.f(vx, s[1]), SVector(vy)) end results = $(g)(df, value(x), value(y)) @@ -296,7 +296,7 @@ for g in (:map, :broadcast) for TX in (:TrackedArray, :TrackedReal), TY in (:TrackedArray, :TrackedReal) TX == :TrackedReal && TY == :TrackedReal && continue - @eval function Base.$(g)(f::ForwardOptimize{F}, x::$(TX){X,D}, y::$(TY){Y,D}) where {F,X,Y,D} + @eval function Base.$(g)(f::ForwardOptimize{F}, x::$(TX){X, D}, y::$(TY){Y, D}) where {F, X, Y, D} result = DiffResults.GradientResult(SVector(zero(D), zero(D))) df = (vx, vy) -> ForwardDiff.gradient!(result, s -> f.f(s[1], s[2]), SVector(vx, vy)) results = $(g)(df, value(x), value(y)) @@ -311,8 +311,8 @@ end for R in REAL_TYPES @eval begin - @inline Base.broadcast(f::ForwardOptimize{F}, x::TrackedArray{X,D}, y::$R) where {F,X,D} = broadcast(ForwardOptimize(t -> f.f(t, y)), x) - @inline Base.broadcast(f::ForwardOptimize{F}, x::$R, y::TrackedArray{Y,D}) where {F,Y,D} = broadcast(ForwardOptimize(t -> f.f(x, t)), y) + @inline Base.broadcast(f::ForwardOptimize{F}, x::TrackedArray{X, D}, y::$R) where {F, X, D} = broadcast(ForwardOptimize(t -> f.f(t, y)), x) + @inline Base.broadcast(f::ForwardOptimize{F}, x::$R, y::TrackedArray{Y, D}) where {F, Y, D} = broadcast(ForwardOptimize(t -> f.f(x, t)), y) end end @@ -411,34 +411,36 @@ end # built-in infix operations # ############################# -const TrackedType = Union{TrackedArray,TrackedReal} +const TrackedType = Union{TrackedArray, TrackedReal} # dispatch # #----------# -for (F, broadcast_f) in ((typeof(+), :broadcast_plus), - (typeof(-), :broadcast_minus), - (typeof(*), :broadcast_mul), - (typeof(/), :broadcast_rdiv), - (typeof(\), :broadcast_ldiv), - (typeof(^), :broadcast_pow)) +for (F, broadcast_f) in ( + (typeof(+), :broadcast_plus), + (typeof(-), :broadcast_minus), + (typeof(*), :broadcast_mul), + (typeof(/), :broadcast_rdiv), + (typeof(\), :broadcast_ldiv), + (typeof(^), :broadcast_pow), + ) @eval begin - @inline Base.broadcast(::$F, x::TrackedArray{X,D}, y::TrackedArray{Y,D}) where {X,Y,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::TrackedReal{X,D}, y::TrackedArray{Y,D}) where {X,Y,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::TrackedArray{X,D}, y::TrackedReal{Y,D}) where {X,Y,D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedArray{X, D}, y::TrackedArray{Y, D}) where {X, Y, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedReal{X, D}, y::TrackedArray{Y, D}) where {X, Y, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedArray{X, D}, y::TrackedReal{Y, D}) where {X, Y, D} = $(broadcast_f)(x, y, D) end for A in ARRAY_TYPES @eval begin - @inline Base.broadcast(::$F, x::TrackedArray{X,D}, y::$A{<:Real}) where {X,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::$A{<:Real}, y::TrackedArray{Y,D}) where {Y,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::TrackedReal{X,D}, y::$A{<:Real}) where {X,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::$A{<:Real}, y::TrackedReal{Y,D}) where {Y,D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedArray{X, D}, y::$A{<:Real}) where {X, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::$A{<:Real}, y::TrackedArray{Y, D}) where {Y, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedReal{X, D}, y::$A{<:Real}) where {X, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::$A{<:Real}, y::TrackedReal{Y, D}) where {Y, D} = $(broadcast_f)(x, y, D) end end for R in REAL_TYPES @eval begin - @inline Base.broadcast(::$F, x::TrackedArray{X,D}, y::$R) where {X,D} = $(broadcast_f)(x, y, D) - @inline Base.broadcast(::$F, x::$R, y::TrackedArray{Y,D}) where {Y,D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::TrackedArray{X, D}, y::$R) where {X, D} = $(broadcast_f)(x, y, D) + @inline Base.broadcast(::$F, x::$R, y::TrackedArray{Y, D}) where {Y, D} = $(broadcast_f)(x, y, D) end end end @@ -446,7 +448,7 @@ end # .+ # #----# -function broadcast_plus(x, y, ::Type{D}) where D +function broadcast_plus(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) .+ value(y), D, tp) cache = (index_bound(x, out), index_bound(y, out)) @@ -454,7 +456,7 @@ function broadcast_plus(x, y, ::Type{D}) where D return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(+)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(+)}}) a, b = instruction.input output = instruction.output pull_value!(a) @@ -463,7 +465,7 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(+)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(+)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) @@ -477,7 +479,7 @@ end # .- # #----# -function broadcast_minus(x, y, ::Type{D}) where D +function broadcast_minus(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) .- value(y), D, tp) cache = (index_bound(x, out), index_bound(y, out)) @@ -485,7 +487,7 @@ function broadcast_minus(x, y, ::Type{D}) where D return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(-)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(-)}}) a, b = instruction.input output = instruction.output pull_value!(a) @@ -494,7 +496,7 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(-)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(-)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) @@ -508,7 +510,7 @@ end # .* # #----# -function broadcast_mul(x, y, ::Type{D}) where D +function broadcast_mul(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) .* value(y), D, tp) cache = (index_bound(x, out), index_bound(y, out)) @@ -516,7 +518,7 @@ function broadcast_mul(x, y, ::Type{D}) where D return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(*)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(*)}}) a, b = instruction.input output = instruction.output pull_value!(a) @@ -525,7 +527,7 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(*)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(*)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) @@ -543,17 +545,19 @@ denom_partials_kernel(n::Real, d::Real) = -(n / d) / d denom_partials(n, d) = broadcast(denom_partials_kernel, n, d) denom_partials!(out::AbstractArray, n, d) = (broadcast!(denom_partials_kernel, out, n, d); nothing) -function broadcast_rdiv(x, y, ::Type{D}) where D +function broadcast_rdiv(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) ./ value(y), D, tp) d_partials = denom_partials(value(x), value(y)) - cache = (d_partials, index_bound(x, out), index_bound(y, out), - index_bound(d_partials, out)) + cache = ( + d_partials, index_bound(x, out), index_bound(y, out), + index_bound(d_partials, out), + ) record!(tp, SpecialInstruction, (broadcast, /), (x, y), out, cache) return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(/)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(/)}}) a, b = instruction.input a_value, b_value = value(a), value(b) d_partials = first(instruction.cache) @@ -565,7 +569,7 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(/)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(/)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) @@ -579,17 +583,19 @@ end # .\ # #----# -function broadcast_ldiv(x, y, ::Type{D}) where D +function broadcast_ldiv(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) .\ value(y), D, tp) d_partials = denom_partials(value(y), value(x)) - cache = (d_partials, index_bound(x, out), index_bound(y, out), - index_bound(d_partials, out)) + cache = ( + d_partials, index_bound(x, out), index_bound(y, out), + index_bound(d_partials, out), + ) record!(tp, SpecialInstruction, (broadcast, \), (x, y), out, cache) return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(\)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(\)}}) a, b = instruction.input a_value, b_value = value(a), value(b) d_partials = first(instruction.cache) @@ -601,7 +607,7 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(\)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(\)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) @@ -629,18 +635,20 @@ function pow_cache(x, y) return (pow_x, pow_y) end -function broadcast_pow(x, y, ::Type{D}) where D +function broadcast_pow(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) .^ value(y), D, tp) bs_partials, ex_partials = pow_cache(x, y) - cache = (bs_partials, ex_partials, - index_bound(x, out), index_bound(y, out), - index_bound(bs_partials, out), index_bound(ex_partials, out)) + cache = ( + bs_partials, ex_partials, + index_bound(x, out), index_bound(y, out), + index_bound(bs_partials, out), index_bound(ex_partials, out), + ) record!(tp, SpecialInstruction, (broadcast, ^), (x, y), out, cache) return out end -@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(^)}}) +@noinline function special_forward_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(^)}}) a, b = instruction.input a_value, b_value = value(a), value(b) bs_partials, ex_partials = instruction.cache @@ -653,12 +661,12 @@ end return nothing end -@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast),typeof(^)}}) +@noinline function special_reverse_exec!(instruction::SpecialInstruction{Tuple{typeof(broadcast), typeof(^)}}) a, b = instruction.input output = instruction.output output_deriv = deriv(output) bs_partials, ex_partials, a_bound, b_bound, - bs_partials_bound, ex_partials_bound = instruction.cache + bs_partials_bound, ex_partials_bound = instruction.cache istracked(a) && broadcast_increment_deriv!(a, output_deriv, bs_partials, a_bound, bs_partials_bound) istracked(b) && broadcast_increment_deriv!(b, output_deriv, ex_partials, b_bound, ex_partials_bound) unseed!(output) diff --git a/src/derivatives/linalg/arithmetic.jl b/src/derivatives/linalg/arithmetic.jl index 4fde5205..c1f1ef56 100644 --- a/src/derivatives/linalg/arithmetic.jl +++ b/src/derivatives/linalg/arithmetic.jl @@ -13,16 +13,16 @@ end # Base allocating version -@inline Base.:+(x::TrackedArray{X,D}, y::TrackedArray{Y,D}) where {X,Y,D} = record_plus(x, y, D) +@inline Base.:+(x::TrackedArray{X, D}, y::TrackedArray{Y, D}) where {X, Y, D} = record_plus(x, y, D) -@inline Base.:+(x::TrackedArray{V,D}, y::AbstractArray) where {V,D} = record_plus(x, y, D) -@inline Base.:+(x::AbstractArray, y::TrackedArray{V,D}) where {V,D} = record_plus(x, y, D) +@inline Base.:+(x::TrackedArray{V, D}, y::AbstractArray) where {V, D} = record_plus(x, y, D) +@inline Base.:+(x::AbstractArray, y::TrackedArray{V, D}) where {V, D} = record_plus(x, y, D) # avoid ambiguities with StaticArrays -@inline Base.:+(x::TrackedArray{V,D}, y::StaticArray) where {V,D} = record_plus(x, y, D) -@inline Base.:+(x::StaticArray, y::TrackedArray{V,D}) where {V,D} = record_plus(x, y, D) +@inline Base.:+(x::TrackedArray{V, D}, y::StaticArray) where {V, D} = record_plus(x, y, D) +@inline Base.:+(x::StaticArray, y::TrackedArray{V, D}) where {V, D} = record_plus(x, y, D) -function record_plus(x, y, ::Type{D}) where D +function record_plus(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) + value(y), D, tp) record!(tp, SpecialInstruction, +, (x, y), out) @@ -59,33 +59,35 @@ function minus!(out, x) for i in eachindex(out) out[i] = -(x[i]) end + return end function minus!(out, x, y) for i in eachindex(out) out[i] = x[i] - y[i] end + return end # Base allocating version -@inline Base.:-(x::TrackedArray{X,D}, y::TrackedArray{Y,D}) where {X,Y,D} = record_minus(x, y, D) +@inline Base.:-(x::TrackedArray{X, D}, y::TrackedArray{Y, D}) where {X, Y, D} = record_minus(x, y, D) -@inline Base.:-(x::TrackedArray{V,D}, y::AbstractArray) where {V,D} = record_minus(x, y, D) -@inline Base.:-(x::AbstractArray, y::TrackedArray{V,D}) where {V,D} = record_minus(x, y, D) +@inline Base.:-(x::TrackedArray{V, D}, y::AbstractArray) where {V, D} = record_minus(x, y, D) +@inline Base.:-(x::AbstractArray, y::TrackedArray{V, D}) where {V, D} = record_minus(x, y, D) # avoid ambiguities with StaticArrays -@inline Base.:-(x::TrackedArray{V,D}, y::StaticArray) where {V,D} = record_minus(x, y, D) -@inline Base.:-(x::StaticArray, y::TrackedArray{V,D}) where {V,D} = record_minus(x, y, D) +@inline Base.:-(x::TrackedArray{V, D}, y::StaticArray) where {V, D} = record_minus(x, y, D) +@inline Base.:-(x::StaticArray, y::TrackedArray{V, D}) where {V, D} = record_minus(x, y, D) -function Base.:-(x::TrackedArray{V,D}) where {V,D} +function Base.:-(x::TrackedArray{V, D}) where {V, D} tp = tape(x) out = track(-(value(x)), D, tp) record!(tp, SpecialInstruction, -, x, out) return out end -function record_minus(x, y, ::Type{D}) where D +function record_minus(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(value(x) - value(y), D, tp) record!(tp, SpecialInstruction, -, (x, y), out) @@ -137,10 +139,10 @@ mulargpullvalue!(x::Transpose) = pull_value!(transpose(x)) # derivatives w.r.t. structured matrices retain their structure function structured_mul!( - C::AbstractVecOrMat, - A::Union{Number,AbstractVecOrMat}, - B::Union{Number,AbstractVecOrMat}, -) + C::AbstractVecOrMat, + A::Union{Number, AbstractVecOrMat}, + B::Union{Number, AbstractVecOrMat}, + ) return mul!(C, A, B) end function structured_mul!(C::Diagonal, A::AbstractVecOrMat, B::AbstractVecOrMat) @@ -148,10 +150,10 @@ function structured_mul!(C::Diagonal, A::AbstractVecOrMat, B::AbstractVecOrMat) return C end function structured_mul!( - C::Union{UpperTriangular,LowerTriangular,UnitUpperTriangular,UnitLowerTriangular}, - A::AbstractVecOrMat, - B::AbstractVecOrMat, -) + C::Union{UpperTriangular, LowerTriangular, UnitUpperTriangular, UnitLowerTriangular}, + A::AbstractVecOrMat, + B::AbstractVecOrMat, + ) mul!(parent(C), A, B) return C end @@ -159,7 +161,7 @@ end # recording pass # #----------------# -@inline function record_mul(x, y, ::Type{D}) where D +@inline function record_mul(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(*(mulargvalue(x), mulargvalue(y)), D, tp) cache = (similar(x, D), similar(y, D)) @@ -167,7 +169,7 @@ end return out end -@inline function record_mul!(out::TrackedArray{V,D}, x, y) where {V,D} +@inline function record_mul!(out::TrackedArray{V, D}, x, y) where {V, D} copyto!(mulargvalue(out), *(mulargvalue(x), mulargvalue(y))) cache = (similar(x, D), similar(y, D)) record!(tape(x, y), SpecialInstruction, *, (x, y), out, cache) @@ -177,55 +179,55 @@ end for S1 in (:TrackedArray, :TrackedVector, :TrackedMatrix) for S2 in (:TrackedArray, :TrackedVector, :TrackedMatrix) @eval begin - LinearAlgebra.:*(x::$(S1){X,D}, y::$(S2){Y,D}) where {X,Y,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){X, D}, y::$(S2){Y, D}) where {X, Y, D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Transpose{<:TrackedReal,<:$(S1){X,D}}, y::Transpose{<:TrackedReal,<:$(S2){Y,D}}) where {X,Y,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Adjoint{<:TrackedReal,<:$(S1){X,D}}, y::Adjoint{<:TrackedReal,<:$(S2){Y,D}}) where {X,Y,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Transpose{<:TrackedReal, <:$(S1){X, D}}, y::Transpose{<:TrackedReal, <:$(S2){Y, D}}) where {X, Y, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Adjoint{<:TrackedReal, <:$(S1){X, D}}, y::Adjoint{<:TrackedReal, <:$(S2){Y, D}}) where {X, Y, D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Transpose{<:TrackedReal,<:$(S1){X,D}}, y::$(S2){Y,D}) where {X,Y,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(S1){X,D}, y::Transpose{<:TrackedReal,<:$(S2){Y,D}}) where {X,Y,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Adjoint{<:TrackedReal,<:$(S1){X,D}}, y::$(S2){Y,D}) where {X,Y,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(S1){X,D}, y::Adjoint{<:TrackedReal,<:$(S2){Y,D}}) where {X,Y,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Transpose{<:TrackedReal, <:$(S1){X, D}}, y::$(S2){Y, D}) where {X, Y, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){X, D}, y::Transpose{<:TrackedReal, <:$(S2){Y, D}}) where {X, Y, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Adjoint{<:TrackedReal, <:$(S1){X, D}}, y::$(S2){Y, D}) where {X, Y, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){X, D}, y::Adjoint{<:TrackedReal, <:$(S2){Y, D}}) where {X, Y, D} = record_mul(x, y, D) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::$(S1){X,D}, y::$(S2){Y,D}) where {V,X,Y,D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::$(S1){X, D}, y::$(S2){Y, D}) where {V, X, Y, D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::Transpose{<:TrackedReal,<:$(S1){X,D}}, y::Transpose{<:TrackedReal,<:$(S2){Y,D}}) where {V,X,Y,D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::Adjoint{<:TrackedReal,<:$(S1){X,D}}, y::Adjoint{<:Number,<:$(S2){Y,D}}) where {V,X,Y,D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::Transpose{<:TrackedReal, <:$(S1){X, D}}, y::Transpose{<:TrackedReal, <:$(S2){Y, D}}) where {V, X, Y, D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::Adjoint{<:TrackedReal, <:$(S1){X, D}}, y::Adjoint{<:Number, <:$(S2){Y, D}}) where {V, X, Y, D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::Transpose{<:TrackedReal,<:$(S1){X,D}}, y::$(S2){Y,D}) where {V,X,Y,D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::$(S1){X,D}, y::Transpose{<:TrackedReal,<:$(S2){Y,D}}) where {V,X,Y,D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::Adjoint{<:TrackedReal,<:$(S1){X,D}}, y::$(S2){Y,D}) where {V,X,Y,D} = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray{V,D}, x::$(S1){X,D}, y::Adjoint{<:TrackedReal,<:$(S2){Y,D}}) where {V,X,Y,D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::Transpose{<:TrackedReal, <:$(S1){X, D}}, y::$(S2){Y, D}) where {V, X, Y, D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::$(S1){X, D}, y::Transpose{<:TrackedReal, <:$(S2){Y, D}}) where {V, X, Y, D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::Adjoint{<:TrackedReal, <:$(S1){X, D}}, y::$(S2){Y, D}) where {V, X, Y, D} = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray{V, D}, x::$(S1){X, D}, y::Adjoint{<:TrackedReal, <:$(S2){Y, D}}) where {V, X, Y, D} = record_mul!(out, x, y) end end for T in ARRAY_TYPES @eval begin - LinearAlgebra.:*(x::$(S1){V,D}, y::$(T)) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(T), y::$(S1){V,D}) where {V,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){V, D}, y::$(T)) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(T), y::$(S1){V, D}) where {V, D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Transpose{<:Number,<:$(T)}, y::$(S1){V,D}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(S1){V,D}, y::Transpose{<:Number,<:$(T)}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Adjoint{<:Number,<:$(T)}, y::$(S1){V,D}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(S1){V,D}, y::Adjoint{<:Number,<:$(T)}) where {V,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Transpose{<:Number, <:$(T)}, y::$(S1){V, D}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){V, D}, y::Transpose{<:Number, <:$(T)}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Adjoint{<:Number, <:$(T)}, y::$(S1){V, D}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(S1){V, D}, y::Adjoint{<:Number, <:$(T)}) where {V, D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Transpose{<:Number,<:$(S1){V,D}}, y::$(T)) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(T), y::Transpose{<:Number,<:$(S1){V,D}}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::Adjoint{<:Number,<:$(S1){V,D}}, y::$(T)) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$(T), y::Adjoint{<:Number,<:$(S1){V,D}}) where {V,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Transpose{<:Number, <:$(S1){V, D}}, y::$(T)) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(T), y::Transpose{<:Number, <:$(S1){V, D}}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Adjoint{<:Number, <:$(S1){V, D}}, y::$(T)) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$(T), y::Adjoint{<:Number, <:$(S1){V, D}}) where {V, D} = record_mul(x, y, D) LinearAlgebra.mul!(out::TrackedArray, x::$(S1), y::$(T)) = record_mul!(out, x, y) LinearAlgebra.mul!(out::TrackedArray, x::$(T), y::$(S1)) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::$(S1), y::Transpose{<:Number,<:$(T)}) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::Transpose{<:Number,<:$(T)}, y::$(S1)) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::$(S1), y::Adjoint{<:Number,<:$(T)}) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::Adjoint{<:Number,<:$(T)}, y::$(S1)) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::$(S1), y::Transpose{<:Number, <:$(T)}) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::Transpose{<:Number, <:$(T)}, y::$(S1)) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::$(S1), y::Adjoint{<:Number, <:$(T)}) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::Adjoint{<:Number, <:$(T)}, y::$(S1)) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::Transpose{<:Number,<:$(S1)}, y::$(T)) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::$(T), y::Transpose{<:Number,<:$(S1)}) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::Adjoint{<:Number,<:$(S1)}, y::$(T)) = record_mul!(out, x, y) - LinearAlgebra.mul!(out::TrackedArray, x::$(T), y::Adjoint{<:Number,<:$(S1)}) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::Transpose{<:Number, <:$(S1)}, y::$(T)) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::$(T), y::Transpose{<:Number, <:$(S1)}) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::Adjoint{<:Number, <:$(S1)}, y::$(T)) = record_mul!(out, x, y) + LinearAlgebra.mul!(out::TrackedArray, x::$(T), y::Adjoint{<:Number, <:$(S1)}) = record_mul!(out, x, y) end end end @@ -234,17 +236,17 @@ end @static if VERSION < v"1.11" for W in (:Diagonal, :(LinearAlgebra.AbstractTriangular)) @eval begin - LinearAlgebra.:*(x::$W, y::TrackedVector{V,D}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::$W, y::TrackedMatrix{V,D}) where {V,D} = record_mul(x, y, D) - LinearAlgebra.:*(x::TrackedMatrix{V,D}, y::$W) where {V,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$W, y::TrackedVector{V, D}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::$W, y::TrackedMatrix{V, D}) where {V, D} = record_mul(x, y, D) + LinearAlgebra.:*(x::TrackedMatrix{V, D}, y::$W) where {V, D} = record_mul(x, y, D) end end else - LinearAlgebra.:*(x::Diagonal, y::TrackedVector{V,D}) where {V,D} = record_mul(x, y, D) + LinearAlgebra.:*(x::Diagonal, y::TrackedVector{V, D}) where {V, D} = record_mul(x, y, D) end # avoid ambiguities with StaticArrays -LinearAlgebra.:*(x::StaticMatrix, y::TrackedVector{V,D}) where {V,D} = record_mul(x, y, D) +LinearAlgebra.:*(x::StaticMatrix, y::TrackedVector{V, D}) where {V, D} = record_mul(x, y, D) # forward pass # #--------------# @@ -289,6 +291,7 @@ function reverse_mul!(output, output_deriv, a, b, a_tmp, b_tmp) end end istracked(b) && increment_deriv!(b, structured_mul!(b_tmp, transpose(value(a)), output_deriv)) + return nothing end ## zero diff --git a/src/derivatives/linalg/reductions.jl b/src/derivatives/linalg/reductions.jl index dbdb5ca3..feba9742 100644 --- a/src/derivatives/linalg/reductions.jl +++ b/src/derivatives/linalg/reductions.jl @@ -5,7 +5,7 @@ # basic sum # #-----------# -function Base.sum(x::TrackedArray{V,D}) where {V,D} +function Base.sum(x::TrackedArray{V, D}) where {V, D} tp = tape(x) out = track(sum(value(x)), D, tp) record!(tp, SpecialInstruction, sum, x, out) @@ -38,7 +38,7 @@ end Base.sum!(y::TrackedArray, x::TrackedArray) = record_sum!(y, x) Base.sum!(y::TrackedArray, x::AbstractArray) = record_sum!(y, x) -function Base.sum(x::TrackedArray{V,D}, dims) where {V,D} +function Base.sum(x::TrackedArray{V, D}, dims) where {V, D} tp = tape(x) out = track(sum(value(x), dims), D, tp) record!(tp, SpecialInstruction, sum!, x, out, index_bound(out, x)) @@ -63,7 +63,7 @@ end # dot # ####### -function record_dot(x, y, ::Type{D}) where D +function record_dot(x, y, ::Type{D}) where {D} tp = tape(x, y) out = track(dot(value(x), value(y)), D, tp) cache = (similar(x, D), similar(y, D)) @@ -71,11 +71,11 @@ function record_dot(x, y, ::Type{D}) where D return out end -LinearAlgebra.dot(x::TrackedArray{X,D}, y::TrackedArray{Y,D}) where {X,Y,D} = record_dot(x, y, D) +LinearAlgebra.dot(x::TrackedArray{X, D}, y::TrackedArray{Y, D}) where {X, Y, D} = record_dot(x, y, D) for A in ARRAY_TYPES - @eval LinearAlgebra.dot(x::TrackedArray{X,D}, y::$A) where {X,D} = record_dot(x, y, D) - @eval LinearAlgebra.dot(x::$A, y::TrackedArray{Y,D}) where {Y,D} = record_dot(x, y, D) + @eval LinearAlgebra.dot(x::TrackedArray{X, D}, y::$A) where {X, D} = record_dot(x, y, D) + @eval LinearAlgebra.dot(x::$A, y::TrackedArray{Y, D}) where {Y, D} = record_dot(x, y, D) end @noinline function special_forward_exec!(instruction::SpecialInstruction{typeof(dot)}) diff --git a/src/derivatives/linalg/special.jl b/src/derivatives/linalg/special.jl index 8de10803..d1fa62db 100644 --- a/src/derivatives/linalg/special.jl +++ b/src/derivatives/linalg/special.jl @@ -18,7 +18,7 @@ end # det # ####### -function LinearAlgebra.det(x::TrackedArray{V,D}) where {V,D} +function LinearAlgebra.det(x::TrackedArray{V, D}) where {V, D} tp = tape(x) x_value = value(x) det_x_value = det(x_value) @@ -52,7 +52,7 @@ end # inv # ####### -function LinearAlgebra.inv(x::TrackedArray{V,D}) where {V,D} +function LinearAlgebra.inv(x::TrackedArray{V, D}) where {V, D} tp = tape(x) out_value = inv(value(x)) out = track(out_value, D, tp) diff --git a/src/derivatives/propagation.jl b/src/derivatives/propagation.jl index 570a18b0..ab9ca6f0 100644 --- a/src/derivatives/propagation.jl +++ b/src/derivatives/propagation.jl @@ -1,4 +1,3 @@ - #= The code here is mainly deals with propagating derivative information between input and output values. Usually, this means incrementing/decrementing the input's derivative(s) by an @@ -20,9 +19,9 @@ efficiency. # utilities # ############# -index_bound(x::Any, ::AbstractArray{T,N}) where {T,N} = nothing +index_bound(x::Any, ::AbstractArray{T, N}) where {T, N} = nothing -index_bound(x::AbstractArray, ::AbstractArray{T,N}) where {T,N} = CartesianIndex{N}(ntuple(i -> size(x, i), Val(N))) +index_bound(x::AbstractArray, ::AbstractArray{T, N}) where {T, N} = CartesianIndex{N}(ntuple(i -> size(x, i), Val(N))) ################### # increment_deriv # @@ -96,19 +95,23 @@ end # diffresult_increment_deriv! # ############################### -@inline getpartial(r::DiffResults.ImmutableDiffResult{1,V,Tuple{D}}, p) where {V,D<:AbstractArray} = DiffResults.derivative(r)[p] -@inline getpartial(r::DiffResults.ImmutableDiffResult{1,V,Tuple{D}}, p) where {V,D<:Number} = DiffResults.derivative(r) +@inline getpartial(r::DiffResults.ImmutableDiffResult{1, V, Tuple{D}}, p) where {V, D <: AbstractArray} = DiffResults.derivative(r)[p] +@inline getpartial(r::DiffResults.ImmutableDiffResult{1, V, Tuple{D}}, p) where {V, D <: Number} = DiffResults.derivative(r) -function diffresult_increment_deriv!(input::AbstractArray, x::AbstractArray, - results, p::Int) +function diffresult_increment_deriv!( + input::AbstractArray, x::AbstractArray, + results, p::Int + ) for i in eachindex(x, results) increment_deriv!(input, x[i] * getpartial(results[i], p), i) end return nothing end -function diffresult_increment_deriv!(input::AbstractArray, x::AbstractArray, - results, p::Int, bound::CartesianIndex) +function diffresult_increment_deriv!( + input::AbstractArray, x::AbstractArray, + results, p::Int, bound::CartesianIndex + ) axes(x) == axes(results) || throw(DimensionMismatch("`x` and `results` must have the same indices")) for (xi, r, i) in zip(x, results, CartesianIndices(size(x))) @@ -117,8 +120,10 @@ function diffresult_increment_deriv!(input::AbstractArray, x::AbstractArray, return nothing end -function diffresult_increment_deriv!(input::TrackedReal, x::AbstractArray, - results, p::Int, ::Nothing) +function diffresult_increment_deriv!( + input::TrackedReal, x::AbstractArray, + results, p::Int, ::Nothing + ) inds = eachindex(x, results) isempty(inds) && return nothing pull_deriv!(input) @@ -134,8 +139,10 @@ end # without partials # #------------------# -function broadcast_increment_deriv!(input::AbstractArray, x::AbstractArray, - bound::CartesianIndex) +function broadcast_increment_deriv!( + input::AbstractArray, x::AbstractArray, + bound::CartesianIndex + ) for (xi, i) in zip(x, CartesianIndices(size(x))) increment_deriv!(input, xi, min(bound, i)) end @@ -162,10 +169,12 @@ end # with partial array # #--------------------# -function _broadcast_increment_deriv!(op::F, input::AbstractArray, x::AbstractArray, - partials::AbstractArray, - input_bound::CartesianIndex, - partials_bound::CartesianIndex) where {F} +function _broadcast_increment_deriv!( + op::F, input::AbstractArray, x::AbstractArray, + partials::AbstractArray, + input_bound::CartesianIndex, + partials_bound::CartesianIndex + ) where {F} for (xi, i) in zip(x, CartesianIndices(size(x))) current_deriv = op(xi, partials[min(partials_bound, i)]) increment_deriv!(input, current_deriv, min(input_bound, i)) @@ -173,9 +182,11 @@ function _broadcast_increment_deriv!(op::F, input::AbstractArray, x::AbstractArr return nothing end -function _broadcast_increment_deriv!(op::F, input::TrackedReal, x::AbstractArray, - partials::AbstractArray, ::Nothing, - ::CartesianIndex) where {F} +function _broadcast_increment_deriv!( + op::F, input::TrackedReal, x::AbstractArray, + partials::AbstractArray, ::Nothing, + ::CartesianIndex + ) where {F} inds = eachindex(x, partials) isempty(inds) && return nothing pull_deriv!(input) @@ -187,9 +198,11 @@ end # with partial scalar # #---------------------# -function _broadcast_increment_deriv!(op::F, input::AbstractArray, x::AbstractArray, - partial::Real, input_bound::CartesianIndex, - ::Nothing) where {F} +function _broadcast_increment_deriv!( + op::F, input::AbstractArray, x::AbstractArray, + partial::Real, input_bound::CartesianIndex, + ::Nothing + ) where {F} for (xi, i) in zip(x, CartesianIndices(size(x))) increment_deriv!(input, op(xi, partial), min(input_bound, i)) end @@ -200,8 +213,10 @@ end # broadcast_decrement_deriv! # ############################## -function broadcast_decrement_deriv!(input::AbstractArray, x::AbstractArray, - bound::CartesianIndex) +function broadcast_decrement_deriv!( + input::AbstractArray, x::AbstractArray, + bound::CartesianIndex + ) for (xi, i) in zip(x, CartesianIndices(size(x))) decrement_deriv!(input, xi, min(bound, i)) end @@ -220,8 +235,10 @@ end # reduction_increment_deriv! # ############################## -function reduction_increment_deriv!(input::AbstractArray, x::AbstractArray, - bound::CartesianIndex) +function reduction_increment_deriv!( + input::AbstractArray, x::AbstractArray, + bound::CartesianIndex + ) for i in CartesianIndices(size(input)) increment_deriv!(input, x[min(bound, i)], i) end diff --git a/src/derivatives/scalars.jl b/src/derivatives/scalars.jl index 9d94f85d..dd09551a 100644 --- a/src/derivatives/scalars.jl +++ b/src/derivatives/scalars.jl @@ -2,7 +2,7 @@ # ForwardOptimize # ################### -for (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) +for (M, f, arity) in DiffRules.diffrules(; filter_modules = nothing) if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f)) @warn "$M.$f is not available and hence rule for it can not be defined" continue # Skip rules for methods not defined in the current scope @@ -49,7 +49,7 @@ end # reverse # ########### -@noinline function scalar_reverse_exec!(instruction::ScalarInstruction{F,I,O,C}) where {F,I,O,C} +@noinline function scalar_reverse_exec!(instruction::ScalarInstruction{F, I, O, C}) where {F, I, O, C} f = instruction.func input = instruction.input output = instruction.output @@ -77,7 +77,7 @@ end # forward # ########### -@noinline function scalar_forward_exec!(instruction::ScalarInstruction{F,I,O,C}) where {F,I,O,C} +@noinline function scalar_forward_exec!(instruction::ScalarInstruction{F, I, O, C}) where {F, I, O, C} f = instruction.func input = instruction.input output = instruction.output @@ -90,7 +90,7 @@ end return nothing end -@noinline function unary_scalar_forward_exec!(f::F, output::O, input, cache) where {F,O} +@noinline function unary_scalar_forward_exec!(f::F, output::O, input, cache) where {F, O} pull_value!(input) result1 = DiffResult(zero(valtype(O)), zero(valtype(O))) result1 = ForwardDiff.derivative!(result1, f, value(input)) @@ -99,7 +99,7 @@ end return nothing end -@noinline function binary_scalar_forward_exec!(f::F, output::O, input, cache) where {F,O} +@noinline function binary_scalar_forward_exec!(f::F, output::O, input, cache) where {F, O} a, b = input pull_value!(a) pull_value!(b) diff --git a/src/macros.jl b/src/macros.jl index 5e15acd1..083e83ee 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -81,7 +81,7 @@ end # unary # #-------# -@inline function (self::ForwardOptimize{F})(t::TrackedReal{V,D}) where {F,V,D} +@inline function (self::ForwardOptimize{F})(t::TrackedReal{V, D}) where {F, V, D} T = promote_type(V, D) result = DiffResult(zero(T), zero(T)) result = ForwardDiff.derivative!(result, self.f, value(t)) @@ -95,7 +95,7 @@ end # binary # #--------# -@inline function (self::ForwardOptimize{F})(a::TrackedReal{V1,D}, b::TrackedReal{V2,D}) where {F,V1,V2,D} +@inline function (self::ForwardOptimize{F})(a::TrackedReal{V1, D}, b::TrackedReal{V2, D}) where {F, V1, V2, D} T = promote_type(V1, V2, D) result = DiffResults.GradientResult(SVector(zero(T), zero(T))) result = ForwardDiff.gradient!(result, x -> self.f(x[1], x[2]), SVector(value(a), value(b))) @@ -106,7 +106,7 @@ end return out end -@inline function (self::ForwardOptimize{F})(x::Real, t::TrackedReal{V,D}) where {F,V,D} +@inline function (self::ForwardOptimize{F})(x::Real, t::TrackedReal{V, D}) where {F, V, D} T = promote_type(typeof(x), V, D) result = DiffResult(zero(T), zero(T)) result = ForwardDiff.derivative!(result, vt -> self.f(x, vt), value(t)) @@ -118,7 +118,7 @@ end return out end -@inline function (self::ForwardOptimize{F})(t::TrackedReal{V,D}, x::Real) where {F,V,D} +@inline function (self::ForwardOptimize{F})(t::TrackedReal{V, D}, x::Real) where {F, V, D} T = promote_type(typeof(x), V, D) result = DiffResult(zero(T), zero(T)) result = ForwardDiff.derivative!(result, vt -> self.f(vt, x), value(t)) @@ -130,8 +130,8 @@ end return out end -@inline (self::ForwardOptimize{F})(x::Dual, t::TrackedReal) where {F} = invoke(self.f, Tuple{Dual,Real}, x, t) -@inline (self::ForwardOptimize{F})(t::TrackedReal, x::Dual) where {F} = invoke(self.f, Tuple{Real,Dual}, t, x) +@inline (self::ForwardOptimize{F})(x::Dual, t::TrackedReal) where {F} = invoke(self.f, Tuple{Dual, Real}, x, t) +@inline (self::ForwardOptimize{F})(t::TrackedReal, x::Dual) where {F} = invoke(self.f, Tuple{Real, Dual}, t, x) ################################# # Skip Instruction Optimization # @@ -187,7 +187,7 @@ macro grad(expr) args_ex = getargs_expr(d[:args]) kwargs_ex = getkwargs_expr(d[:kwargs]) return quote - function $ReverseDiff.track(::typeof($f), $(d[:args]...); $(d[:kwargs]...)) where {$(d[:whereparams]...),} + function $ReverseDiff.track(::typeof($f), $(d[:args]...); $(d[:kwargs]...)) where {$(d[:whereparams]...)} $closure_ex $args = $args_ex $kwargs = $kwargs_ex @@ -206,9 +206,9 @@ macro grad(expr) end if !hasmethod( - $ReverseDiff.special_reverse_exec!, - Tuple{$ReverseDiff.SpecialInstruction{typeof($f)}}, - ) + $ReverseDiff.special_reverse_exec!, + Tuple{$ReverseDiff.SpecialInstruction{typeof($f)}}, + ) @noinline function $ReverseDiff.special_reverse_exec!(instruction::$ReverseDiff.SpecialInstruction{typeof($f)}) output = instruction.output input = instruction.input @@ -222,9 +222,9 @@ macro grad(expr) end if !hasmethod( - $ReverseDiff.special_forward_exec!, - Tuple{$ReverseDiff.SpecialInstruction{typeof($f)}}, - ) + $ReverseDiff.special_forward_exec!, + Tuple{$ReverseDiff.SpecialInstruction{typeof($f)}}, + ) @noinline function $ReverseDiff.special_forward_exec!(instruction::$ReverseDiff.SpecialInstruction{typeof($f)}) output, input = instruction.output, instruction.input $ReverseDiff.pull_value!.(input) diff --git a/src/tape.jl b/src/tape.jl index afaf8770..be8e9323 100644 --- a/src/tape.jl +++ b/src/tape.jl @@ -6,16 +6,18 @@ abstract type AbstractInstruction end const InstructionTape = Vector{AbstractInstruction} -function record!(tp::InstructionTape, ::Type{InstructionType}, args...) where InstructionType +function record!(tp::InstructionTape, ::Type{InstructionType}, args...) where {InstructionType} tp !== NULL_TAPE && push!(tp, InstructionType(args...)) return nothing end function Base.:(==)(a::AbstractInstruction, b::AbstractInstruction) - return (a.func == b.func && + return ( + a.func == b.func && a.input == b.input && a.output == b.output && - a.cache == b.cache) + a.cache == b.cache + ) end # Ensure that the external state is "captured" so that external @@ -27,19 +29,19 @@ end # ScalarInstruction # #-------------------# -struct ScalarInstruction{F,I,O,C} <: AbstractInstruction +struct ScalarInstruction{F, I, O, C} <: AbstractInstruction func::F input::I output::O cache::C # disable default outer constructor - function ScalarInstruction{F,I,O,C}(func, input, output, cache) where {F,I,O,C} - return new{F,I,O,C}(func, input, output, cache) + function ScalarInstruction{F, I, O, C}(func, input, output, cache) where {F, I, O, C} + return new{F, I, O, C}(func, input, output, cache) end end -@inline function _ScalarInstruction(func::F, input::I, output::O, cache::C) where {F,I,O,C} - return ScalarInstruction{F,I,O,C}(func, input, output, cache) +@inline function _ScalarInstruction(func::F, input::I, output::O, cache::C) where {F, I, O, C} + return ScalarInstruction{F, I, O, C}(func, input, output, cache) end function ScalarInstruction(func, input, output, cache = nothing) @@ -49,19 +51,19 @@ end # SpecialInstruction # #--------------------# -struct SpecialInstruction{F,I,O,C} <: AbstractInstruction +struct SpecialInstruction{F, I, O, C} <: AbstractInstruction func::F input::I output::O cache::C # disable default outer constructor - function SpecialInstruction{F,I,O,C}(func, input, output, cache) where {F,I,O,C} - return new{F,I,O,C}(func, input, output, cache) + function SpecialInstruction{F, I, O, C}(func, input, output, cache) where {F, I, O, C} + return new{F, I, O, C}(func, input, output, cache) end end -@inline function _SpecialInstruction(func::F, input::I, output::O, cache::C) where {F,I,O,C} - return SpecialInstruction{F,I,O,C}(func, input, output, cache) +@inline function _SpecialInstruction(func::F, input::I, output::O, cache::C) where {F, I, O, C} + return SpecialInstruction{F, I, O, C}(func, input, output, cache) end function SpecialInstruction(func, input, output, cache = nothing) @@ -97,7 +99,7 @@ end ################### # extra spaces here accomodates padding in show(::IO, ::AbstractInstruction) -compactrepr(x::Tuple) = "("*join(map(compactrepr, x), ",\n ")*")" +compactrepr(x::Tuple) = "(" * join(map(compactrepr, x), ",\n ") * ")" compactrepr(x::AbstractArray) = length(x) < 5 ? match(r"\[.*?\]", repr(x)).match : summary(x) compactrepr(x) = repr(x) @@ -106,7 +108,8 @@ function Base.show(io::IO, instruction::AbstractInstruction, pad = "") println(io, pad, "$(name)($(instruction.func)):") println(io, pad, " input: ", compactrepr(instruction.input)) println(io, pad, " output: ", compactrepr(instruction.output)) - print(io, pad, " cache: ", compactrepr(instruction.cache)) + print(io, pad, " cache: ", compactrepr(instruction.cache)) + return nothing end function Base.show(io::IO, tp::InstructionTape) @@ -118,4 +121,5 @@ function Base.show(io::IO, tp::InstructionTape) println(io) i += 1 end + return end diff --git a/src/tracked.jl b/src/tracked.jl index af140909..8f9c7f2f 100644 --- a/src/tracked.jl +++ b/src/tracked.jl @@ -44,51 +44,55 @@ overcomplicates the API and would incur unneccesary pointer loads during re-vali making the below implementation preferable. =# -mutable struct TrackedReal{V<:Real,D<:Real,O} <: Real +mutable struct TrackedReal{V <: Real, D <: Real, O} <: Real value::V deriv::D tape::InstructionTape index::Int origin::O - TrackedReal{V,D,O}(value, deriv, tape, index, origin) where {V,D,O} = new{V,D,O}(value, deriv, tape, index, origin) - TrackedReal{V,D,O}(value, deriv, tape) where {V,D,O} = new{V,D,O}(value, deriv, tape, NULL_INDEX) - TrackedReal{V,D,O}(value, deriv) where {V,D,O} = new{V,D,O}(value, deriv, NULL_TAPE, NULL_INDEX) - TrackedReal{V,D,O}(value) where {V,D,O} = new{V,D,O}(value, zero(D), NULL_TAPE, NULL_INDEX) + TrackedReal{V, D, O}(value, deriv, tape, index, origin) where {V, D, O} = new{V, D, O}(value, deriv, tape, index, origin) + TrackedReal{V, D, O}(value, deriv, tape) where {V, D, O} = new{V, D, O}(value, deriv, tape, NULL_INDEX) + TrackedReal{V, D, O}(value, deriv) where {V, D, O} = new{V, D, O}(value, deriv, NULL_TAPE, NULL_INDEX) + TrackedReal{V, D, O}(value) where {V, D, O} = new{V, D, O}(value, zero(D), NULL_TAPE, NULL_INDEX) end -TrackedReal(v::V, a::D, tp::InstructionTape, i::Int, o::O) where {V,D,O} = TrackedReal{V,D,O}(v, a, tp, i, o) +TrackedReal(v::V, a::D, tp::InstructionTape, i::Int, o::O) where {V, D, O} = TrackedReal{V, D, O}(v, a, tp, i, o) -TrackedReal(v::V, a::D, tp::InstructionTape = NULL_TAPE) where {V,D} = TrackedReal{V,D,Nothing}(v, a, tp) +TrackedReal(v::V, a::D, tp::InstructionTape = NULL_TAPE) where {V, D} = TrackedReal{V, D, Nothing}(v, a, tp) # we define these special cases so that the "constructor <--> convert" pun holds for `TrackedReal` # this is Jarett's favorite piece of code. A true work of art. -@inline TrackedReal{V,D,O}(x::TrackedReal) where {V,D,O} = convert(TrackedReal{V,D,O}, x) +@inline TrackedReal{V, D, O}(x::TrackedReal) where {V, D, O} = convert(TrackedReal{V, D, O}, x) # TrackedArray # #--------------# -struct TrackedArray{V,D,N,VA,DA} <: AbstractArray{TrackedReal{V,D,TrackedArray{V,D,N,VA,DA}},N} +struct TrackedArray{V, D, N, VA, DA} <: AbstractArray{TrackedReal{V, D, TrackedArray{V, D, N, VA, DA}}, N} value::VA deriv::DA tape::InstructionTape - function TrackedArray{V,D,N,VA,DA}(value::AbstractArray{V,N}, - deriv::AbstractArray{D,N}, - tape::InstructionTape) where {V,D,N,VA,DA} + function TrackedArray{V, D, N, VA, DA}( + value::AbstractArray{V, N}, + deriv::AbstractArray{D, N}, + tape::InstructionTape + ) where {V, D, N, VA, DA} @assert IndexStyle(value) === IndexLinear() @assert size(value) === size(deriv) - return new{V,D,N,VA,DA}(value, deriv, tape) + return new{V, D, N, VA, DA}(value, deriv, tape) end end -function TrackedArray(value::AbstractArray{V,N}, - deriv::AbstractArray{D,N}, - tape::InstructionTape) where {V,D,N} - return TrackedArray{V,D,N,typeof(value),typeof(deriv)}(value, deriv, tape) +function TrackedArray( + value::AbstractArray{V, N}, + deriv::AbstractArray{D, N}, + tape::InstructionTape + ) where {V, D, N} + return TrackedArray{V, D, N, typeof(value), typeof(deriv)}(value, deriv, tape) end -const TrackedVector{V,D} = TrackedArray{V,D,1} -const TrackedMatrix{V,D} = TrackedArray{V,D,2} -const TrackedVecOrMat{V,D} = Union{TrackedVector{V,D}, TrackedMatrix{V,D}} +const TrackedVector{V, D} = TrackedArray{V, D, 1} +const TrackedMatrix{V, D} = TrackedArray{V, D, 2} +const TrackedVecOrMat{V, D} = Union{TrackedVector{V, D}, TrackedMatrix{V, D}} ########### # getters # @@ -108,20 +112,20 @@ end @inline value(t::TrackedArray) = t.value @inline deriv(t::TrackedArray) = t.deriv -@inline deriv(t::TrackedReal) = t.deriv +@inline deriv(t::TrackedReal) = t.deriv @inline valtype(::TrackedReal{V}) where {V} = V -@inline valtype(::Type{TrackedReal{V,D,O}}) where {V,D,O} = V +@inline valtype(::Type{TrackedReal{V, D, O}}) where {V, D, O} = V @inline valtype(::TrackedArray{V}) where {V} = V -@inline valtype(::Type{TrackedArray{V,D,N,VA,DA}}) where {V,D,VA,DA,N} = V +@inline valtype(::Type{TrackedArray{V, D, N, VA, DA}}) where {V, D, VA, DA, N} = V -@inline derivtype(::TrackedReal{V,D}) where {V,D} = D -@inline derivtype(::Type{TrackedReal{V,D,O}}) where {V,D,O} = D -@inline derivtype(t::TrackedArray{V,D}) where {V,D} = D -@inline derivtype(::Type{TrackedArray{V,D,N,VA,DA}}) where {V,D,VA,DA,N} = D +@inline derivtype(::TrackedReal{V, D}) where {V, D} = D +@inline derivtype(::Type{TrackedReal{V, D, O}}) where {V, D, O} = D +@inline derivtype(t::TrackedArray{V, D}) where {V, D} = D +@inline derivtype(::Type{TrackedArray{V, D, N, VA, DA}}) where {V, D, VA, DA, N} = D -@inline origintype(::TrackedReal{V,D,O}) where {V,D,O} = O -@inline origintype(::Type{TrackedReal{V,D,O}}) where {V,D,O} = O +@inline origintype(::TrackedReal{V, D, O}) where {V, D, O} = O +@inline origintype(::Type{TrackedReal{V, D, O}}) where {V, D, O} = O @inline hasorigin(x::Real) = false @inline hasorigin(t::TrackedReal) = t.index !== NULL_INDEX @@ -158,7 +162,7 @@ end @inline value!(t::TrackedReal, v::Real) = (t.value = v; nothing) @inline value!(t::TrackedArray, v::AbstractArray) = (copyto!(value(t), v); nothing) -function value!(t::NTuple{N,Any}, v::NTuple{N,Any}) where N +function value!(t::NTuple{N, Any}, v::NTuple{N, Any}) where {N} for i in eachindex(t) value!(t[i], v[i]) end @@ -168,7 +172,7 @@ end @inline deriv!(t::TrackedReal, v::Real) = (t.deriv = v; nothing) @inline deriv!(t::TrackedArray, v::AbstractArray) = (copyto!(deriv(t), v); nothing) -function deriv!(t::NTuple{N,Any}, v::NTuple{N,Any}) where N +function deriv!(t::NTuple{N, Any}, v::NTuple{N, Any}) where {N} for i in eachindex(t) deriv!(t[i], v[i]) end @@ -225,7 +229,7 @@ unseed!(x::AbstractArray, i) = unseed!(x[i]) # `forward_pass!`/`reverse_pass!`. capture(t::TrackedReal) = ifelse(hastape(t), t, value(t)) capture(t::TrackedArray) = t -capture(t::AbstractArray) = istracked(t) ? map!(capture, similar(t), t) : copy(t) +capture(t::AbstractArray) = istracked(t) ? map!(capture, similar(t), t) : copy(t) ######################## # Conversion/Promotion # @@ -237,10 +241,10 @@ _convert(::Type{R}, t::Real) where {R <: Real} = R(t) _convert(::Type{R}, t::TrackedReal) where {R <: TrackedReal} = R(t) # recording a instruction for this preserves the line of references back to the origin's deriv -function Base.convert(::Type{T1}, t::T2) where {T1<:TrackedReal,T2<:TrackedReal} +function Base.convert(::Type{T1}, t::T2) where {T1 <: TrackedReal, T2 <: TrackedReal} V1, D1, O1 = valtype(T1), derivtype(T1), origintype(T1) tp = tape(t) - out = TrackedReal{V1,D1,O1}(_convert(V1, value(t)), _convert(D1, deriv(t)), tp) + out = TrackedReal{V1, D1, O1}(_convert(V1, value(t)), _convert(D1, deriv(t)), tp) record!(tp, SpecialInstruction, convert, t, out) return out end @@ -259,37 +263,37 @@ end return nothing end -Base.convert(::Type{Real}, t::T) where {T<:TrackedReal} = t -function Base.convert(::Type{R}, t::T) where {R<:Real,T<:TrackedReal} +Base.convert(::Type{Real}, t::T) where {T <: TrackedReal} = t +function Base.convert(::Type{R}, t::T) where {R <: Real, T <: TrackedReal} throw(ArgumentError("Converting an instance of $T to $R is not defined. Please use `ReverseDiff.value` instead.")) end -function Base.convert(::Type{T}, x::R) where {T<:TrackedReal, R<:Real} +function Base.convert(::Type{T}, x::R) where {T <: TrackedReal, R <: Real} return TrackedReal{valtype(T), derivtype(T), origintype(T)}( convert(valtype(T), value(x)), ) end -Base.convert(::Type{T}, t::T) where {T<:TrackedReal} = t -Base.convert(::Type{T}, t::T) where {T<:TrackedArray} = t +Base.convert(::Type{T}, t::T) where {T <: TrackedReal} = t +Base.convert(::Type{T}, t::T) where {T <: TrackedArray} = t for R in REAL_TYPES - R === :Dual && continue # ForwardDiff.Dual is handled below + R === :Dual && continue # ForwardDiff.Dual is handled below @eval begin if isconcretetype($R) # issue ForwardDiff#322 - Base.promote_rule(::Type{TrackedReal{V,D,O}}, ::Type{$R}) where {V,D,O} = TrackedReal{promote_type(V,$R),D,O} - Base.promote_rule(::Type{$R}, ::Type{TrackedReal{V,D,O}}) where {V,D,O} = TrackedReal{promote_type($R,V),D,O} + Base.promote_rule(::Type{TrackedReal{V, D, O}}, ::Type{$R}) where {V, D, O} = TrackedReal{promote_type(V, $R), D, O} + Base.promote_rule(::Type{$R}, ::Type{TrackedReal{V, D, O}}) where {V, D, O} = TrackedReal{promote_type($R, V), D, O} else - Base.promote_rule(::Type{TrackedReal{V,D,O}}, ::Type{R}) where {V,D,O,R<:$R} = TrackedReal{promote_type(V,R),D,O} - Base.promote_rule(::Type{R}, ::Type{TrackedReal{V,D,O}}) where {R<:$R,V,D,O,} = TrackedReal{promote_type(R,V),D,O} + Base.promote_rule(::Type{TrackedReal{V, D, O}}, ::Type{R}) where {V, D, O, R <: $R} = TrackedReal{promote_type(V, R), D, O} + Base.promote_rule(::Type{R}, ::Type{TrackedReal{V, D, O}}) where {R <: $R, V, D, O} = TrackedReal{promote_type(R, V), D, O} end end end # Avoid method ambiguities for ForwardDiff.Dual -Base.promote_rule(::Type{TrackedReal{V1,D,O}}, ::Type{Dual{T,V2,N}}) where {V1,D,O,T,V2,N} = TrackedReal{promote_type(V1,Dual{T,V2,N}),D,O} -Base.promote_rule(::Type{Dual{T,V1,N}}, ::Type{TrackedReal{V2,D,O}}) where {T,V1,N,V2,D,O} = TrackedReal{promote_type(Dual{T,V1,N},V2),D,O} +Base.promote_rule(::Type{TrackedReal{V1, D, O}}, ::Type{Dual{T, V2, N}}) where {V1, D, O, T, V2, N} = TrackedReal{promote_type(V1, Dual{T, V2, N}), D, O} +Base.promote_rule(::Type{Dual{T, V1, N}}, ::Type{TrackedReal{V2, D, O}}) where {T, V1, N, V2, D, O} = TrackedReal{promote_type(Dual{T, V1, N}, V2), D, O} -Base.promote_rule(::Type{TrackedReal{V1,D1,O1}}, ::Type{TrackedReal{V2,D2,O2}}) where {V1,V2,D1,D2,O1,O2} = TrackedReal{promote_type(V1,V2),promote_type(D1,D2),Nothing} +Base.promote_rule(::Type{TrackedReal{V1, D1, O1}}, ::Type{TrackedReal{V2, D2, O2}}) where {V1, V2, D1, D2, O1, O2} = TrackedReal{promote_type(V1, V2), promote_type(D1, D2), Nothing} ########################### # AbstractArray Interface # @@ -298,7 +302,7 @@ Base.promote_rule(::Type{TrackedReal{V1,D1,O1}}, ::Type{TrackedReal{V2,D2,O2}}) colon2range(s, i) = i colon2range(s, ::Colon) = s -function index_iterable(shape::NTuple{N,Any}, i::NTuple{M,Any}) where {N,M} +function index_iterable(shape::NTuple{N, Any}, i::NTuple{M, Any}) where {N, M} if N < M return index_iterable(shape, ntuple(n -> i[n], Val(N))) elseif M < N && isa(last(i), Colon) @@ -308,7 +312,7 @@ function index_iterable(shape::NTuple{N,Any}, i::NTuple{M,Any}) where {N,M} end end -for T in (:AbstractRange, :Colon, :(Union{Colon,AbstractRange})) +for T in (:AbstractRange, :Colon, :(Union{Colon, AbstractRange})) @eval Base.@propagate_inbounds function Base.getindex(t::TrackedArray, i1::$(T), is::$(T)...) tp = tape(t) out = TrackedArray(value(t)[i1, is...], deriv(t)[i1, is...], tp) @@ -393,7 +397,7 @@ end # a `TrackedArray` backed by views of the parent's buffers shares its deriv array with the # parent, so derivatives accumulate into the parent and nothing has to be recorded (#281) -const ViewIndex = Union{Integer,Colon,AbstractRange,AbstractArray{<:Integer},AbstractArray{<:CartesianIndex}} +const ViewIndex = Union{Integer, Colon, AbstractRange, AbstractArray{<:Integer}, AbstractArray{<:CartesianIndex}} Base.@propagate_inbounds function Base.view(t::TrackedArray, inds::ViewIndex...) v = view(value(t), inds...) @@ -416,11 +420,11 @@ Base.similar(t::TrackedArray, T::Type, args::Union{Integer, AbstractUnitRange}.. reshape_body = :(TrackedArray(reshape(value(t), dims), reshape(deriv(t), dims), tape(t))) @eval Base.reshape(t::TrackedArray, dims::Val{N}) where {N} = $reshape_body -@eval Base.reshape(t::TrackedArray, dims::Tuple{Vararg{Int,N}}) where {N} = $reshape_body +@eval Base.reshape(t::TrackedArray, dims::Tuple{Vararg{Int, N}}) where {N} = $reshape_body @eval Base.reshape(t::TrackedArray, dims::Int64...) = $reshape_body @eval Base.reshape(t::TrackedArray, dims::AbstractUnitRange...) = $reshape_body @eval Base.reshape(t::TrackedArray, dims::Colon...) = $reshape_body -@eval Base.reshape(t::TrackedArray, dims::Union{AbstractUnitRange,Int64,Colon}...) = $reshape_body +@eval Base.reshape(t::TrackedArray, dims::Union{AbstractUnitRange, Int64, Colon}...) = $reshape_body #################### # `Real` Interface # @@ -429,27 +433,27 @@ reshape_body = :(TrackedArray(reshape(value(t), dims), reshape(deriv(t), dims), Base.hash(t::TrackedReal) = hash(value(t)) Base.hash(t::TrackedReal, hsh::UInt64) = hash(value(t), hsh) -Base.deepcopy(t::T) where {T<:TrackedReal} = t -Base.copy(t::T) where {T<:TrackedReal} = t +Base.deepcopy(t::T) where {T <: TrackedReal} = t +Base.copy(t::T) where {T <: TrackedReal} = t Base.float(t::TrackedReal) = _float(t, float(value(t))) -Base.float(t::TrackedReal{V}) where {V<:AbstractFloat} = t +Base.float(t::TrackedReal{V}) where {V <: AbstractFloat} = t Base.float(t::TrackedArray) = _float(t, float(value(t))) -Base.float(t::TrackedArray{V}) where {V<:AbstractFloat} = t +Base.float(t::TrackedArray{V}) where {V <: AbstractFloat} = t # `float` left the value type alone, so there is nothing to record -_float(t::TrackedReal{V}, ::V) where {V<:Real} = t +_float(t::TrackedReal{V}, ::V) where {V <: Real} = t _float(t::TrackedArray{V}, ::AbstractArray{V}) where {V} = t -function _float(t::TrackedReal{V,D}, v::Real) where {V,D} +function _float(t::TrackedReal{V, D}, v::Real) where {V, D} tp = tape(t) out = track(v, D, tp) record!(tp, SpecialInstruction, float, t, out) return out end -function _float(t::TrackedArray{V,D}, v::AbstractArray) where {V,D} +function _float(t::TrackedArray{V, D}, v::AbstractArray) where {V, D} tp = tape(t) out = track(v, D, tp) record!(tp, SpecialInstruction, float, t, out) @@ -470,27 +474,27 @@ end return nothing end -Base.one(::Type{TrackedReal{V,D,O}}) where {V,D,O} = TrackedReal{V,D,O}(one(V)) -Base.zero(::Type{TrackedReal{V,D,O}}) where {V,D,O} = TrackedReal{V,D,O}(zero(V)) +Base.one(::Type{TrackedReal{V, D, O}}) where {V, D, O} = TrackedReal{V, D, O}(one(V)) +Base.zero(::Type{TrackedReal{V, D, O}}) where {V, D, O} = TrackedReal{V, D, O}(zero(V)) # neither identity depends on the origin (#172) -Base.one(::Type{TrackedReal{V,D}}) where {V,D} = TrackedReal{V,D,Nothing}(one(V)) -Base.zero(::Type{TrackedReal{V,D}}) where {V,D} = TrackedReal{V,D,Nothing}(zero(V)) +Base.one(::Type{TrackedReal{V, D}}) where {V, D} = TrackedReal{V, D, Nothing}(one(V)) +Base.zero(::Type{TrackedReal{V, D}}) where {V, D} = TrackedReal{V, D, Nothing}(zero(V)) -Base.rand(::Type{TrackedReal{V,D,O}}) where {V,D,O} = TrackedReal{V,D,O}(rand(V)) -Base.rand(rng::Random.AbstractRNG, ::Type{TrackedReal{V,D,O}}) where {V,D,O} = TrackedReal{V,D,O}(rand(rng, V)) +Base.rand(::Type{TrackedReal{V, D, O}}) where {V, D, O} = TrackedReal{V, D, O}(rand(V)) +Base.rand(rng::Random.AbstractRNG, ::Type{TrackedReal{V, D, O}}) where {V, D, O} = TrackedReal{V, D, O}(rand(rng, V)) Base.eps(t::TrackedReal) = eps(value(t)) -Base.eps(::Type{T}) where {T<:TrackedReal} = eps(valtype(T)) +Base.eps(::Type{T}) where {T <: TrackedReal} = eps(valtype(T)) -Base.floatmax(::Type{T}) where {T<:TrackedReal} = floatmax(valtype(T)) -Base.floatmin(::Type{T}) where {T<:TrackedReal} = floatmin(valtype(T)) +Base.floatmax(::Type{T}) where {T <: TrackedReal} = floatmax(valtype(T)) +Base.floatmin(::Type{T}) where {T <: TrackedReal} = floatmin(valtype(T)) Base.floor(t::TrackedReal) = floor(value(t)) -Base.floor(::Type{R}, t::TrackedReal) where {R<:Real} = floor(R, value(t)) +Base.floor(::Type{R}, t::TrackedReal) where {R <: Real} = floor(R, value(t)) Base.ceil(t::TrackedReal) = ceil(value(t)) -Base.ceil(::Type{R}, t::TrackedReal) where {R<:Real} = ceil(R, value(t)) +Base.ceil(::Type{R}, t::TrackedReal) where {R <: Real} = ceil(R, value(t)) Base.fld(a::TrackedReal, b::TrackedReal) = fld(value(a), value(b)) @@ -499,15 +503,15 @@ Base.cld(a::TrackedReal, b::TrackedReal) = cld(value(a), value(b)) Base.div(x::TrackedReal, y::TrackedReal, r::RoundingMode) = div(value(x), value(y), r) Base.trunc(t::TrackedReal) = trunc(value(t)) -Base.trunc(::Type{R}, t::TrackedReal) where {R<:Real} = trunc(R, value(t)) +Base.trunc(::Type{R}, t::TrackedReal) where {R <: Real} = trunc(R, value(t)) Base.round(t::TrackedReal) = round(value(t)) -Base.round(::Type{R}, t::TrackedReal) where {R<:Real} = round(R, value(t)) +Base.round(::Type{R}, t::TrackedReal) where {R <: Real} = round(R, value(t)) Base.oneunit(t::TrackedReal) = one(t) -Base.oneunit(::Type{T}) where {T<:TrackedReal} = one(T) +Base.oneunit(::Type{T}) where {T <: TrackedReal} = one(T) -Base.rtoldefault(::Type{T}) where {T<:TrackedReal} = sqrt(eps(T)) +Base.rtoldefault(::Type{T}) where {T <: TrackedReal} = sqrt(eps(T)) ################ # track/track! # @@ -534,7 +538,7 @@ track!(t::TrackedArray, x::AbstractArray) = (value!(t, x); unseed!(t); t) track!(t::TrackedReal, x::Real) = (value!(t, x); unseed!(t); t) -function track!(t::AbstractArray{TrackedReal{D,D,Nothing}}, x::AbstractArray, tp::InstructionTape) where D +function track!(t::AbstractArray{TrackedReal{D, D, Nothing}}, x::AbstractArray, tp::InstructionTape) where {D} for i in eachindex(t) t[i] = track(x[i], D, tp) end @@ -545,11 +549,12 @@ end # Pretty Printing # ################### -idstr(x) = string(objectid(x), base=62)[1:3] +idstr(x) = string(objectid(x), base = 62)[1:3] function Base.show(io::IO, t::TrackedReal) tape_id = hastape(t) ? idstr(t.tape) : "---" origin_id = hasorigin(t) ? "$(t.index), $(idstr(t.origin))" : "---" id = idstr(t) print(io, "TrackedReal<$(id)>($(value(t)), $(deriv(t)), $(tape_id), $(origin_id))") + return nothing end diff --git a/test/ChainRulesTests.jl b/test/ChainRulesTests.jl index dce2305d..ffb6a9f9 100644 --- a/test/ChainRulesTests.jl +++ b/test/ChainRulesTests.jl @@ -66,13 +66,13 @@ ReverseDiff.@grad_from_chainrules g(x::ReverseDiff.TrackedArray, y::ReverseDiff. ## ChainRules # function f input = rand(3, 3) - output, back = ChainRulesCore.rrule(f, input); + output, back = ChainRulesCore.rrule(f, input) _, d = back(1) @test output == f(input) @test d == fill(3, size(input)) # function g inputs = rand(3, 3), rand(3, 3) - output, back = ChainRulesCore.rrule(g, inputs...); + output, back = ChainRulesCore.rrule(g, inputs...) _, d1, d2 = back(1) @test output == g(inputs...) @test d1 == fill(3, size(inputs[1])) @@ -81,7 +81,7 @@ ReverseDiff.@grad_from_chainrules g(x::ReverseDiff.TrackedArray, y::ReverseDiff. ## ReverseDiff #function f - inputs = (rand(3, 3), ) + inputs = (rand(3, 3),) results = (similar(inputs[1]),) f_tape = ReverseDiff.GradientTape(x -> f(x) + 2, (rand(3, 3),)) @@ -104,7 +104,7 @@ ReverseDiff.@grad_from_chainrules g(x::ReverseDiff.TrackedArray, y::ReverseDiff. @test results[1] == fill(3, size(inputs[1])) @test results[2] == fill(5, size(inputs[2])) - results = (similar(inputs[1]), similar(inputs[2]),) + results = (similar(inputs[1]), similar(inputs[2])) compiled_tape = ReverseDiff.CompiledTape(f_tape) ReverseDiff.gradient!(results, compiled_tape, inputs) @test results[1] == fill(3, size(inputs[1])) @@ -114,12 +114,12 @@ end @testset "custom struct input" begin input = rand(3, 3) - output, back = ChainRulesCore.rrule(f, MyStruct(), input); + output, back = ChainRulesCore.rrule(f, MyStruct(), input) _, _, d = back(1) @test output == f(MyStruct(), input) @test d == fill(3, size(input)) - output, back = ChainRulesCore.rrule(f, input, MyStruct()); + output, back = ChainRulesCore.rrule(f, input, MyStruct()) _, d, _ = back(1) @test output == f(input, MyStruct()) @test d == fill(3, size(input)) @@ -152,7 +152,7 @@ end ReverseDiff.@grad_from_chainrules f_vararg(x::ReverseDiff.TrackedArray, args...) @testset "Function with Varargs" begin - inputs = (rand(3, 3), ) + inputs = (rand(3, 3),) results = (similar(inputs[1]),) f_tape = ReverseDiff.GradientTape(x -> f_vararg(x, 1, 2, 3) + 2, (rand(3, 3),)) @@ -163,23 +163,23 @@ end # Vargs and kwargs -f_kw(x, args...; k=1, kwargs...) = sum(4x .+ sum(args) .+ (k + kwargs[:j])) +f_kw(x, args...; k = 1, kwargs...) = sum(4x .+ sum(args) .+ (k + kwargs[:j])) -function ChainRulesCore.rrule(::typeof(f_kw), x, args...; k=1, kwargs...) - r = f_kw(x, args...; k=k, kwargs...) +function ChainRulesCore.rrule(::typeof(f_kw), x, args...; k = 1, kwargs...) + r = f_kw(x, args...; k = k, kwargs...) function back(d) return NoTangent(), fill(3 * d, size(x)) end return r, back end -ReverseDiff.@grad_from_chainrules f_kw(x::ReverseDiff.TrackedArray, args...; k=1, kwargs...) +ReverseDiff.@grad_from_chainrules f_kw(x::ReverseDiff.TrackedArray, args...; k = 1, kwargs...) @testset "Function with Varargs and kwargs" begin - inputs = (rand(3, 3), ) + inputs = (rand(3, 3),) results = (similar(inputs[1]),) - f_tape = ReverseDiff.GradientTape(x -> f_kw(x, 1, 2, 3; k=2, j=3) + 2, (rand(3, 3),)) + f_tape = ReverseDiff.GradientTape(x -> f_kw(x, 1, 2, 3; k = 2, j = 3) + 2, (rand(3, 3),)) ReverseDiff.gradient!(results, f_tape, inputs) @test results[1] == fill(3, size(inputs[1])) @@ -196,7 +196,7 @@ end @testset "ReverseDiff and ChainRules Mixed" begin t(x) = g(x, h(x)) - inputs = (rand(3, 3), ) + inputs = (rand(3, 3),) results = (similar(inputs[1]),) g_tape = ReverseDiff.GradientTape(t, (rand(3, 3),)) @@ -250,37 +250,37 @@ end ### Isolated Scope module IsolatedModuleForTestingScoping -using ChainRulesCore -using ReverseDiff: ReverseDiff, @grad_from_chainrules - -f(x) = sum(4x .+ 1) - -function ChainRulesCore.rrule(::typeof(f), x) - r = f(x) - function back(d) - # return a distinguishable but improper grad - return NoTangent(), fill(3 * d, size(x)) + using ChainRulesCore + using ReverseDiff: ReverseDiff, @grad_from_chainrules + + f(x) = sum(4x .+ 1) + + function ChainRulesCore.rrule(::typeof(f), x) + r = f(x) + function back(d) + # return a distinguishable but improper grad + return NoTangent(), fill(3 * d, size(x)) + end + return r, back end - return r, back -end -@grad_from_chainrules f(x::ReverseDiff.TrackedArray) + @grad_from_chainrules f(x::ReverseDiff.TrackedArray) -module SubModule -using Test -using ReverseDiff: TrackedArray, GradientTape, gradient! -using ..IsolatedModuleForTestingScoping: f -@testset "rrule in Isolated Scope" begin - inputs = (rand(3, 3), ) + module SubModule + using Test + using ReverseDiff: TrackedArray, GradientTape, gradient! + using ..IsolatedModuleForTestingScoping: f + @testset "rrule in Isolated Scope" begin + inputs = (rand(3, 3),) - results = (similar(inputs[1]),) - f_tape = GradientTape(x -> f(x) + 2, (rand(3, 3),)) - gradient!(results, f_tape, inputs) + results = (similar(inputs[1]),) + f_tape = GradientTape(x -> f(x) + 2, (rand(3, 3),)) + gradient!(results, f_tape, inputs) - @test results[1] == fill(3, size(inputs[1])) -end + @test results[1] == fill(3, size(inputs[1])) + end -end # end of SubModule + end # end of SubModule end # end of IsolatedModuleForTestingScoping end diff --git a/test/MacrosTests.jl b/test/MacrosTests.jl index 8250e725..a803f8cb 100644 --- a/test/MacrosTests.jl +++ b/test/MacrosTests.jl @@ -12,25 +12,25 @@ x, a, b = rand(3) # @forward # ############ -f0(x) = 1. / (1. + exp(-x)) +f0(x) = 1.0 / (1.0 + exp(-x)) f0(a, b) = sqrt(a^2 + b^2) -ReverseDiff.@forward f1(x::T) where {T<:Real} = 1. / (1. + exp(-x)) -ReverseDiff.@forward f1(a::A, b::B) where {A,B<:Real} = sqrt(a^2 + b^2) +ReverseDiff.@forward f1(x::T) where {T <: Real} = 1.0 / (1.0 + exp(-x)) +ReverseDiff.@forward f1(a::A, b::B) where {A, B <: Real} = sqrt(a^2 + b^2) -ReverseDiff.@forward f2(x) = 1. / (1. + exp(-x)) +ReverseDiff.@forward f2(x) = 1.0 / (1.0 + exp(-x)) ReverseDiff.@forward f2(a, b) = sqrt(a^2 + b^2) -ReverseDiff.@forward function f3(x::T) where T<:Real - return 1. / (1. + exp(-x)) +ReverseDiff.@forward function f3(x::T) where {T <: Real} + return 1.0 / (1.0 + exp(-x)) end -ReverseDiff.@forward function f3(a::A, b::B) where {A,B<:Real} +ReverseDiff.@forward function f3(a::A, b::B) where {A, B <: Real} return sqrt(a^2 + b^2) end ReverseDiff.@forward function f4(x) - return 1. / (1. + exp(-x)) + return 1.0 / (1.0 + exp(-x)) end ReverseDiff.@forward function f4(a, b) @@ -53,6 +53,7 @@ function test_forward(f, x, tp) @test instruction.output === yt @test instruction.cache[] === partials(dual, 1) empty!(tp) + return nothing end function test_forward(f, a, b, tp) @@ -92,6 +93,7 @@ function test_forward(f, a, b, tp) @test instruction.output === tc @test instruction.cache[] === SVector(partials(dual)...) empty!(tp) + return nothing end for f in (ReverseDiff.@forward(f0), f1, f2, f3, f4, ReverseDiff.@forward(-)) @@ -100,7 +102,7 @@ for f in (ReverseDiff.@forward(f0), f1, f2, f3, f4, ReverseDiff.@forward(-)) test_forward(f, a, b, tp) end -ReverseDiff.@forward f5 = (x) -> 1. / (1. + exp(-x)) +ReverseDiff.@forward f5 = (x) -> 1.0 / (1.0 + exp(-x)) test_println("@forward anonymous functions", f5) test_forward(f5, x, tp) @@ -114,22 +116,22 @@ test_forward(f6, a, b, tp) g0 = f0 -ReverseDiff.@skip g1(x::T) where {T<:Real} = 1. / (1. + exp(-x)) -ReverseDiff.@skip g1(a::A, b::B) where {A,B<:Real} = sqrt(a^2 + b^2) +ReverseDiff.@skip g1(x::T) where {T <: Real} = 1.0 / (1.0 + exp(-x)) +ReverseDiff.@skip g1(a::A, b::B) where {A, B <: Real} = sqrt(a^2 + b^2) -ReverseDiff.@skip g2(x) = 1. / (1. + exp(-x)) +ReverseDiff.@skip g2(x) = 1.0 / (1.0 + exp(-x)) ReverseDiff.@skip g2(a, b) = sqrt(a^2 + b^2) -ReverseDiff.@skip function g3(x::T) where T<:Real - return 1. / (1. + exp(-x)) +ReverseDiff.@skip function g3(x::T) where {T <: Real} + return 1.0 / (1.0 + exp(-x)) end -ReverseDiff.@skip function g3(a::A, b::B) where {A,B<:Real} +ReverseDiff.@skip function g3(a::A, b::B) where {A, B <: Real} return sqrt(a^2 + b^2) end ReverseDiff.@skip function g4(x) - return 1. / (1. + exp(-x)) + return 1.0 / (1.0 + exp(-x)) end ReverseDiff.@skip function g4(a, b) @@ -145,6 +147,7 @@ function test_skip(g, x, tp) yt = g(xt) @test yt === y @test isempty(tp) + return nothing end function test_skip(g, a, b, tp) @@ -165,6 +168,7 @@ function test_skip(g, a, b, tp) tc = g(at, bt) @test tc === c @test isempty(tp) + return nothing end for g in (ReverseDiff.@skip(g0), g1, g2, g3, g4) @@ -173,7 +177,7 @@ for g in (ReverseDiff.@skip(g0), g1, g2, g3, g4) test_skip(g, a, b, tp) end -ReverseDiff.@skip g5 = (x) -> 1. / (1. + exp(-x)) +ReverseDiff.@skip g5 = (x) -> 1.0 / (1.0 + exp(-x)) test_println("@skip anonymous functions", g5) test_skip(g5, x, tp) @@ -189,11 +193,11 @@ using LinearAlgebra using ReverseDiff: @grad, TrackedReal, TrackedVector, TrackedMatrix, TrackedArray @testset "@grad macro" begin - x = rand(3); - A = rand(3, 3); - A_x = [vec(A); x]; + x = rand(3) + A = rand(3, 3) + A_x = [vec(A); x] global custom_grad_called - + f1(x) = dot(x, x) f1(x::TrackedVector) = ReverseDiff.track(f1, x) @grad function f1(x::AbstractVector) @@ -201,13 +205,13 @@ using ReverseDiff: @grad, TrackedReal, TrackedVector, TrackedMatrix, TrackedArra xv = ReverseDiff.value(x) dot(xv, xv), Δ -> (Δ * 2 * xv,) end - + custom_grad_called = false g1 = ReverseDiff.gradient(f1, x) g2 = ReverseDiff.gradient(x -> dot(x, x), x) @test g1 == g2 @test custom_grad_called - + f2(A, x) = A * x f2(A, x::TrackedVector) = ReverseDiff.track(f2, A, x) f2(A::TrackedMatrix, x) = ReverseDiff.track(f2, A, x) @@ -218,19 +222,19 @@ using ReverseDiff: @grad, TrackedReal, TrackedVector, TrackedMatrix, TrackedArra xv = ReverseDiff.value(x) Av * xv, Δ -> (Δ * xv', Av' * Δ) end - + custom_grad_called = false g1 = ReverseDiff.gradient(x -> sum(f2(A, x)), x) g2 = ReverseDiff.gradient(x -> sum(A * x), x) @test g1 == g2 @test custom_grad_called - + custom_grad_called = false g1 = ReverseDiff.gradient(A -> sum(f2(A, x)), A) g2 = ReverseDiff.gradient(A -> sum(A * x), A) @test g1 == g2 @test custom_grad_called - + custom_grad_called = false g1 = ReverseDiff.gradient(A_x -> sum(f2(reshape(A_x[1:9], 3, 3), A_x[10:end])), A_x) g2 = ReverseDiff.gradient(A_x -> sum(reshape(A_x[1:9], 3, 3) * A_x[10:end]), A_x) @@ -289,7 +293,7 @@ using ReverseDiff: @grad, TrackedReal, TrackedVector, TrackedMatrix, TrackedArra g2 = ReverseDiff.gradient(sum, x) @test g1 == g2 @test custom_grad_called - + x2 = round.(Int, x) custom_grad_called = false g1 = ReverseDiff.gradient(f6, x2) diff --git a/test/PrecisionTests.jl b/test/PrecisionTests.jl index 29d777aa..09c7cb90 100644 --- a/test/PrecisionTests.jl +++ b/test/PrecisionTests.jl @@ -11,45 +11,47 @@ setprecision(BigFloat, 256) do T = eltype(x) rtol = 8 * eps(T) - for (f, ∇f) in ((mean, y -> fill(one(T) / n, n)), - (y -> mean(y .^ 3), y -> 3 .* y .^ 2 ./ n), - (y -> sum(y ./ 3), y -> fill(one(T) / 3, n)), - (y -> sum(y ./ 3.0), y -> fill(one(T) / 3, n)), - (y -> sum(3 .\ y), y -> fill(one(T) / 3, n)), - (y -> sum(3.0 .\ y), y -> fill(one(T) / 3, n)), - (sum, y -> fill(one(T), n)), - (prod, y -> prod(y) ./ y), - (norm, y -> y ./ norm(y)), - (var, y -> 2 .* (y .- mean(y)) ./ (n - 1)), - (std, y -> (y .- mean(y)) ./ ((n - 1) * std(y))), - (y -> dot(y, y), y -> 2 .* y), - (y -> sum(3 ./ y), y -> -3 ./ y .^ 2), - (y -> sum(y .\ 3), y -> -3 ./ y .^ 2), - (y -> sum(y ./ (y .+ 1)), y -> 1 ./ (y .+ 1) .^ 2), - (y -> sum(exp.(y)), y -> exp.(y))) + for (f, ∇f) in ( + (mean, y -> fill(one(T) / n, n)), + (y -> mean(y .^ 3), y -> 3 .* y .^ 2 ./ n), + (y -> sum(y ./ 3), y -> fill(one(T) / 3, n)), + (y -> sum(y ./ 3.0), y -> fill(one(T) / 3, n)), + (y -> sum(3 .\ y), y -> fill(one(T) / 3, n)), + (y -> sum(3.0 .\ y), y -> fill(one(T) / 3, n)), + (sum, y -> fill(one(T), n)), + (prod, y -> prod(y) ./ y), + (norm, y -> y ./ norm(y)), + (var, y -> 2 .* (y .- mean(y)) ./ (n - 1)), + (std, y -> (y .- mean(y)) ./ ((n - 1) * std(y))), + (y -> dot(y, y), y -> 2 .* y), + (y -> sum(3 ./ y), y -> -3 ./ y .^ 2), + (y -> sum(y .\ 3), y -> -3 ./ y .^ 2), + (y -> sum(y ./ (y .+ 1)), y -> 1 ./ (y .+ 1) .^ 2), + (y -> sum(exp.(y)), y -> exp.(y)), + ) test_println("BigFloat gradients", f) - @test ReverseDiff.gradient(f, x) ≈ ∇f(x) rtol=rtol + @test ReverseDiff.gradient(f, x) ≈ ∇f(x) rtol = rtol end # fused broadcasts lose precision in `ForwardDiff._div_partials` (upstream) let f = y -> sum(y .^ 2 ./ 7) test_println("BigFloat gradients", f) - @test_broken ReverseDiff.gradient(f, x) ≈ 2 .* x ./ 7 rtol=rtol + @test_broken ReverseDiff.gradient(f, x) ≈ 2 .* x ./ 7 rtol = rtol end end # a subnormal denominator must not overflow the gradient -let f = y -> 1e-10 * sum(y ./ 1e-310) +let f = y -> 1.0e-10 * sum(y ./ 1.0e-310) test_println("Float64 gradients", f) - @test ReverseDiff.gradient(f, [1.0, 2.0]) == fill(1e-10 / 1e-310, 2) + @test ReverseDiff.gradient(f, [1.0, 2.0]) == fill(1.0e-10 / 1.0e-310, 2) end # the denominator adjoint is `-n / d^2`, which must not overflow while it is still finite -for (n, d) in ((1e-200, 1e-200), (1e200, 1e200)) +for (n, d) in ((1.0e-200, 1.0e-200), (1.0e200, 1.0e200)) ∂ = Float64(-big(n) / big(d)^2) for f in (y -> sum([n] ./ y), y -> sum(y .\ [n])) test_println("Float64 gradients", f) - @test ReverseDiff.gradient(f, [d]) ≈ [∂] rtol=8*eps(Float64) + @test ReverseDiff.gradient(f, [d]) ≈ [∂] rtol = 8 * eps(Float64) end end diff --git a/test/TrackedTests.jl b/test/TrackedTests.jl index bdbf8442..b9c12ea9 100644 --- a/test/TrackedTests.jl +++ b/test/TrackedTests.jl @@ -19,16 +19,20 @@ function samefields(a::TrackedReal, b::TrackedReal) same_origin = true end end - return (ReverseDiff.value(a) == ReverseDiff.value(b) && + return ( + ReverseDiff.value(a) == ReverseDiff.value(b) && ReverseDiff.deriv(a) == ReverseDiff.deriv(b) && ReverseDiff.tape(a) === ReverseDiff.tape(b) && - a.index === b.index && same_origin) + a.index === b.index && same_origin + ) end function samefields(a::TrackedArray, b::TrackedArray) - return (ReverseDiff.value(a) == ReverseDiff.value(b) && + return ( + ReverseDiff.value(a) == ReverseDiff.value(b) && ReverseDiff.deriv(a) == ReverseDiff.deriv(b) && - ReverseDiff.tape(a) === ReverseDiff.tape(b)) + ReverseDiff.tape(a) === ReverseDiff.tape(b) + ) end ################ @@ -49,7 +53,7 @@ t = TrackedReal(v, d, tp, 1, o) @test t.tape === tp @test t.index === 1 @test t.origin === o -@test isa(t, TrackedReal{Float64,Int,Vector{Float64}}) +@test isa(t, TrackedReal{Float64, Int, Vector{Float64}}) t = TrackedReal(v, d, tp) @@ -58,7 +62,7 @@ t = TrackedReal(v, d, tp) @test t.tape === tp @test t.index === ReverseDiff.NULL_INDEX @test t.origin === nothing -@test isa(t, TrackedReal{Float64,Int,Nothing}) +@test isa(t, TrackedReal{Float64, Int, Nothing}) t = TrackedReal(v, d) @@ -67,7 +71,7 @@ t = TrackedReal(v, d) @test t.tape === ReverseDiff.NULL_TAPE @test t.index === ReverseDiff.NULL_INDEX @test t.origin === nothing -@test isa(t, TrackedReal{Float64,Int,Nothing}) +@test isa(t, TrackedReal{Float64, Int, Nothing}) # TrackedArray # #--------------# @@ -81,7 +85,7 @@ t = TrackedArray(v, d, tp) @test t.value === v @test t.deriv === d @test t.tape === tp -@test isa(t, TrackedArray{Float64,Int,3,Array{Float64,3},Array{Int,3}}) +@test isa(t, TrackedArray{Float64, Int, 3, Array{Float64, 3}, Array{Int, 3}}) ########### # getters # @@ -106,7 +110,7 @@ t = TrackedArray(v, d, tp) @test ReverseDiff.istracked(Real[1.0]) @test ReverseDiff.istracked(Number[1.0]) @test ReverseDiff.istracked(AbstractVector[[1.0]]) -@test ReverseDiff.istracked(Union{Float64,TrackedReal{Float64,Float64,Nothing}}[1.0]) +@test ReverseDiff.istracked(Union{Float64, TrackedReal{Float64, Float64, Nothing}}[1.0]) # a vector of `TrackedArray`s is tracked, and `value` and `tape` see through it let ta = TrackedArray(rand(2), rand(2), InstructionTape()), v = [ta, ta] @@ -145,9 +149,9 @@ any_varr = Any[v] # valtype # #---------# -@test ReverseDiff.valtype(TrackedReal(1,0)) === Int +@test ReverseDiff.valtype(TrackedReal(1, 0)) === Int @test ReverseDiff.valtype(TrackedReal(1.0, 0)) === Float64 -@test ReverseDiff.valtype(typeof(TrackedReal(1,0))) === Int +@test ReverseDiff.valtype(typeof(TrackedReal(1, 0))) === Int @test ReverseDiff.valtype(typeof(TrackedReal(1.0, 0))) === Float64 @test ReverseDiff.valtype(TrackedArray([1], [0], InstructionTape())) === Int @test ReverseDiff.valtype(TrackedArray([1.0], [0], InstructionTape())) === Float64 @@ -157,9 +161,9 @@ any_varr = Any[v] # derivtype # #-----------# -@test ReverseDiff.derivtype(TrackedReal(1,0)) === Int +@test ReverseDiff.derivtype(TrackedReal(1, 0)) === Int @test ReverseDiff.derivtype(TrackedReal(1, 0.0)) === Float64 -@test ReverseDiff.derivtype(typeof(TrackedReal(1,0))) === Int +@test ReverseDiff.derivtype(typeof(TrackedReal(1, 0))) === Int @test ReverseDiff.derivtype(typeof(TrackedReal(1, 0.0))) === Float64 @test ReverseDiff.derivtype(TrackedArray([1], [0], InstructionTape())) === Int @test ReverseDiff.derivtype(TrackedArray([1], [0.0], InstructionTape())) === Float64 @@ -178,10 +182,14 @@ any_varr = Any[v] tp = InstructionTape() -null_tape_items = (nothing, rand(), [1], Any[1], [TrackedReal(1, 1)], TrackedReal(1, 1), - TrackedArray(rand(1), rand(1), ReverseDiff.NULL_TAPE)) -tape_items = ([3, TrackedReal(1, 1, tp)], TrackedReal(1, 1, tp), - TrackedArray(rand(1), rand(1), tp)) +null_tape_items = ( + nothing, rand(), [1], Any[1], [TrackedReal(1, 1)], TrackedReal(1, 1), + TrackedArray(rand(1), rand(1), ReverseDiff.NULL_TAPE), +) +tape_items = ( + [3, TrackedReal(1, 1, tp)], TrackedReal(1, 1, tp), + TrackedArray(rand(1), rand(1), tp), +) for i in null_tape_items @test ReverseDiff.tape(i) === ReverseDiff.NULL_TAPE @@ -266,7 +274,7 @@ varr_copy = copy(ReverseDiff.value(ta)) trs = Any[tr, ta[1], ta[2]] @test ReverseDiff.pull_value!(nothing) === nothing -@test ReverseDiff.pull_value!([1,2,3]) === nothing +@test ReverseDiff.pull_value!([1, 2, 3]) === nothing @test ReverseDiff.pull_value!(1.0) === nothing ReverseDiff.value(ta)[1] = v_new @@ -308,7 +316,7 @@ darr_copy = copy(ReverseDiff.deriv(ta)) trs = Any[tr, ta[1], ta[2]] @test ReverseDiff.pull_deriv!(nothing) === nothing -@test ReverseDiff.pull_deriv!([1,2,3]) === nothing +@test ReverseDiff.pull_deriv!([1, 2, 3]) === nothing @test ReverseDiff.pull_deriv!(1.0) === nothing ReverseDiff.deriv(ta)[1] = d_new @@ -350,7 +358,7 @@ darr_copy = copy(ReverseDiff.deriv(ta)) trs = Any[tr, ta[1], ta[2]] @test ReverseDiff.push_deriv!(nothing) === nothing -@test ReverseDiff.push_deriv!([1,2,3]) === nothing +@test ReverseDiff.push_deriv!([1, 2, 3]) === nothing @test ReverseDiff.push_deriv!(1.0) === nothing ReverseDiff.deriv!(ta1, d_new) @@ -390,10 +398,10 @@ tr = TrackedReal(v, d) trs = Any[tr, ta[1], ta[2]] @test ReverseDiff.seed!(nothing) === nothing -@test ReverseDiff.seed!([1,2,3]) === nothing +@test ReverseDiff.seed!([1, 2, 3]) === nothing @test ReverseDiff.seed!(1.0) === nothing @test ReverseDiff.unseed!(nothing) === nothing -@test ReverseDiff.unseed!([1,2,3]) === nothing +@test ReverseDiff.unseed!([1, 2, 3]) === nothing @test ReverseDiff.unseed!(1.0) === nothing ReverseDiff.seed!(ta) @@ -600,10 +608,10 @@ tp = InstructionTape() ta = TrackedArray(varr, darr, tp) tr = TrackedReal(v, d) ta1 = ta[1] -A = TrackedArray{BigInt,Float64,1,Array{BigInt,1},Array{Float64,1}} -T = TrackedReal{BigInt,Float64,A} +A = TrackedArray{BigInt, Float64, 1, Array{BigInt, 1}, Array{Float64, 1}} +T = TrackedReal{BigInt, Float64, A} -t2 = convert(TrackedReal{BigFloat,BigFloat,Nothing}, ta1) +t2 = convert(TrackedReal{BigFloat, BigFloat, Nothing}, ta1) @test length(tp) == 1 instr = tp[1] @test instr.func === convert @@ -616,7 +624,7 @@ empty!(tp) @test_throws ArgumentError convert(BigFloat, tr) @test samefields(convert(T, 1), T(big(1), 0.0)) -@test samefields(convert(TrackedReal{BigInt,Float64,Nothing}, 1), TrackedReal(big(1), 0.0)) +@test samefields(convert(TrackedReal{BigInt, Float64, Nothing}, 1), TrackedReal(big(1), 0.0)) @test convert(typeof(tr), tr) === tr @test convert(typeof(ta), ta) === ta @@ -625,18 +633,18 @@ empty!(tp) @test promote_type(T, Bool) === T @test promote_type(T, Int32) === T @test promote_type(T, Int64) === T -@test promote_type(T, Integer) === TrackedReal{BigInt,Float64,A} -@test promote_type(T, typeof(ℯ)) === TrackedReal{BigFloat,Float64,A} -@test promote_type(T, typeof(π)) === TrackedReal{BigFloat,Float64,A} -@test promote_type(T, typeof(log2π)) === TrackedReal{BigFloat,Float64,A} -@test promote_type(T, Rational{Int}) === TrackedReal{Rational{BigInt},Float64,A} -@test promote_type(T, BigFloat) === TrackedReal{BigFloat,Float64,A} +@test promote_type(T, Integer) === TrackedReal{BigInt, Float64, A} +@test promote_type(T, typeof(ℯ)) === TrackedReal{BigFloat, Float64, A} +@test promote_type(T, typeof(π)) === TrackedReal{BigFloat, Float64, A} +@test promote_type(T, typeof(log2π)) === TrackedReal{BigFloat, Float64, A} +@test promote_type(T, Rational{Int}) === TrackedReal{Rational{BigInt}, Float64, A} +@test promote_type(T, BigFloat) === TrackedReal{BigFloat, Float64, A} @test promote_type(T, BigInt) === T -@test promote_type(T, Float64) === TrackedReal{BigFloat,Float64,A} -@test promote_type(T, AbstractFloat) === TrackedReal{BigFloat,Float64,A} -@test promote_type(T, Real) === TrackedReal{Real,Float64,A} -@test promote_type(T, ForwardDiff.Dual{:tag,Float64,1}) === TrackedReal{ForwardDiff.Dual{:tag,BigFloat,1},Float64,A} -@test promote_type(T, TrackedReal{BigFloat,BigFloat,Nothing}) === TrackedReal{BigFloat,BigFloat,Nothing} +@test promote_type(T, Float64) === TrackedReal{BigFloat, Float64, A} +@test promote_type(T, AbstractFloat) === TrackedReal{BigFloat, Float64, A} +@test promote_type(T, Real) === TrackedReal{Real, Float64, A} +@test promote_type(T, ForwardDiff.Dual{:tag, Float64, 1}) === TrackedReal{ForwardDiff.Dual{:tag, BigFloat, 1}, Float64, A} +@test promote_type(T, TrackedReal{BigFloat, BigFloat, Nothing}) === TrackedReal{BigFloat, BigFloat, Nothing} @test promote_type(T, T) === T ########################### @@ -656,7 +664,7 @@ for T in (UInt, Int) @test samefields(ta[S(2), T(3)], TrackedReal(varr[2, 3], darr[2, 3], tp, 8, ta)) end -ta_sub = ta[:,:] +ta_sub = ta[:, :] idx = ReverseDiff.index_iterable(axes(ta), (:, :)) @test collect(idx) == [(i, j) for i in 1:3, j in 1:3] @test samefields(ta_sub, ta) @@ -669,41 +677,41 @@ instr = tp[1] empty!(tp) for T in (UInt, Int) - ta_sub = ta[:,T(1):T(2)] + ta_sub = ta[:, T(1):T(2)] idx = ReverseDiff.index_iterable(axes(ta), (:, T(1):T(2))) @test collect(idx) == [(i, j) for i in 1:3, j in 1:2] - @test samefields(ta_sub, TrackedArray(varr[:,1:2], darr[:,1:2], tp)) + @test samefields(ta_sub, TrackedArray(varr[:, 1:2], darr[:, 1:2], tp)) @test length(tp) == 1 instr = tp[1] @test instr.func === getindex @test instr.input === (ta, idx) - @test samefields(instr.output, TrackedArray(varr[:,1:2], darr[:,1:2], tp)) + @test samefields(instr.output, TrackedArray(varr[:, 1:2], darr[:, 1:2], tp)) @test instr.cache === nothing empty!(tp) - ta_sub = ta[T(2):T(3),:] + ta_sub = ta[T(2):T(3), :] idx = ReverseDiff.index_iterable(axes(ta), (T(2):T(3), :)) @test collect(idx) == [(i, j) for i in 2:3, j in 1:3] - @test samefields(ta_sub, TrackedArray(varr[2:3,:], darr[2:3,:], tp)) + @test samefields(ta_sub, TrackedArray(varr[2:3, :], darr[2:3, :], tp)) @test length(tp) == 1 instr = tp[1] @test instr.func === getindex @test instr.input === (ta, idx) - @test samefields(instr.output, TrackedArray(varr[2:3,:], darr[2:3,:], tp)) + @test samefields(instr.output, TrackedArray(varr[2:3, :], darr[2:3, :], tp)) @test instr.cache === nothing empty!(tp) S = T === UInt ? Int : UInt for U in (S, T) - ta_sub = ta[S(1):S(2),T(2):T(3)] + ta_sub = ta[S(1):S(2), T(2):T(3)] idx = ReverseDiff.index_iterable(axes(ta), (S(1):S(2), T(2):T(3))) @test collect(idx) == [(i, j) for i in 1:2, j in 2:3] - @test samefields(ta_sub, TrackedArray(varr[1:2,2:3], darr[1:2,2:3], tp)) + @test samefields(ta_sub, TrackedArray(varr[1:2, 2:3], darr[1:2, 2:3], tp)) @test length(tp) == 1 instr = tp[1] @test instr.func === getindex @test instr.input === (ta, idx) - @test samefields(instr.output, TrackedArray(varr[1:2,2:3], darr[1:2,2:3], tp)) + @test samefields(instr.output, TrackedArray(varr[1:2, 2:3], darr[1:2, 2:3], tp)) @test instr.cache === nothing empty!(tp) end @@ -808,7 +816,7 @@ tr_float32 = TrackedReal(Float32(v_float), Float32(d), tp) @test deepcopy(tr_float) === tr_float @test copy(tr_float) === tr_float -@test samefields(float(tr_int), TrackedReal{Float64,Float64,Nothing}(float(v_int), zero(d), tp)) +@test samefields(float(tr_int), TrackedReal{Float64, Float64, Nothing}(float(v_int), zero(d), tp)) @test float(tr_float) === tr_float @test float(tr_float32) === tr_float32 @@ -817,14 +825,14 @@ tr_float32 = TrackedReal(Float32(v_float), Float32(d), tp) @test samefields(zero(tr_float), typeof(tr_float)(zero(v_float))) # an eltype whose origin is free still has both identities (#172) -@test samefields(one(TrackedReal{Float64,Float64}), TrackedReal{Float64,Float64,Nothing}(1.0)) -@test samefields(zero(TrackedReal{Float64,Float64}), TrackedReal{Float64,Float64,Nothing}(0.0)) +@test samefields(one(TrackedReal{Float64, Float64}), TrackedReal{Float64, Float64, Nothing}(1.0)) +@test samefields(zero(TrackedReal{Float64, Float64}), TrackedReal{Float64, Float64, Nothing}(0.0)) -tr_rand = rand(TrackedReal{Int,Float64,Nothing}) -@test samefields(tr_rand, TrackedReal{Int,Float64,Nothing}(ReverseDiff.value(tr_rand))) +tr_rand = rand(TrackedReal{Int, Float64, Nothing}) +@test samefields(tr_rand, TrackedReal{Int, Float64, Nothing}(ReverseDiff.value(tr_rand))) -tr_rand = rand(MersenneTwister(1), TrackedReal{Int,Float64,Nothing}) -@test samefields(tr_rand, TrackedReal{Int,Float64,Nothing}(ReverseDiff.value(tr_rand))) +tr_rand = rand(MersenneTwister(1), TrackedReal{Int, Float64, Nothing}) +@test samefields(tr_rand, TrackedReal{Int, Float64, Nothing}(ReverseDiff.value(tr_rand))) @test eps(tr_float) === eps(v_float) @test eps(typeof(tr_float)) === eps(Float64) @@ -866,7 +874,7 @@ end f = rand() tr1 = ReverseDiff.TrackedReal(f, rand(), tp) -tr2 = ReverseDiff.TrackedReal(2*f, rand(), tp) +tr2 = ReverseDiff.TrackedReal(2 * f, rand(), tp) tr3 = ReverseDiff.TrackedReal(f + eps(f), rand(), tp) @test isapprox(tr1, tr2) == false @@ -902,7 +910,7 @@ ta = TrackedArray(varr, darr, tp) trs = similar(ta) tp2 = InstructionTape() @test isa(trs, Vector{eltype(ta)}) -trs = similar(ta, TrackedReal{Float64,Float64,Nothing}) +trs = similar(ta, TrackedReal{Float64, Float64, Nothing}) track!(trs, ta, tp2) for i in eachindex(trs) @test samefields(trs[i], track(varr[i], tp2)) diff --git a/test/api/ConfigTests.jl b/test/api/ConfigTests.jl index e37005c9..d5e4e507 100644 --- a/test/api/ConfigTests.jl +++ b/test/api/ConfigTests.jl @@ -39,19 +39,19 @@ for Config in (GradientConfig, JacobianConfig) end cfg = JacobianConfig(z, x, tp) -zt = similar(z, ReverseDiff.TrackedReal{eltype(z),eltype(z),Nothing}) +zt = similar(z, ReverseDiff.TrackedReal{eltype(z), eltype(z), Nothing}) @test issimilar(cfg.input, track(x, eltype(z), tp)) @test issimilar(cfg.output, track!(zt, z, tp)) @test cfg.tape === tp cfg = JacobianConfig(z, (x, y), tp) -zt = similar(z, ReverseDiff.TrackedReal{eltype(z),eltype(z),Nothing}) +zt = similar(z, ReverseDiff.TrackedReal{eltype(z), eltype(z), Nothing}) @test issimilar(cfg.output, track!(zt, z, tp)) @test issimilar(cfg.input, (track(x, eltype(z), tp), track(y, eltype(z), tp))) @test cfg.tape === tp cfg1 = JacobianConfig(z, (x, y), tp) -zt = similar(z, ReverseDiff.TrackedReal{eltype(z),eltype(z),Nothing}) +zt = similar(z, ReverseDiff.TrackedReal{eltype(z), eltype(z), Nothing}) cfg2 = JacobianConfig(DiffResults.JacobianResult(z), (x, y), tp) @test issimilar(cfg1, cfg2) diff --git a/test/api/GradientTests.jl b/test/api/GradientTests.jl index 1f4db4ef..76d2b4d5 100644 --- a/test/api/GradientTests.jl +++ b/test/api/GradientTests.jl @@ -67,6 +67,7 @@ function test_unary_gradient(f, x) test_approx(DiffResults.value(result), DiffResults.value(test)) test_approx(DiffResults.gradient(result), DiffResults.gradient(test)) end + return nothing end function test_ternary_gradient(f, a, b, c) @@ -170,6 +171,7 @@ function test_ternary_gradient(f, a, b, c) test_approx(DiffResults.gradient(∇b), test_b) test_approx(DiffResults.gradient(∇c), test_c) end + return nothing end # issue https://github.com/JuliaDiff/ReverseDiff.jl/issues/140 @@ -193,10 +195,12 @@ norm_hermitian2(v) = (A = I - 2 * v * transpose(v); norm(transpose(A) * A)) norm_hermitian3(v) = (A = I - 2 * v * collect(v'); norm(collect(A') * A)) norm_hermitian4(v) = (A = I - 2 * v * v'; norm(transpose(A) * A)) norm_hermitian5(v) = (A = I - 2 * v * transpose(v); norm(A' * A)) -norm_hermitian6(v) = (A = (v'v)*I - 2 * v * v'; norm(A' * A)) +norm_hermitian6(v) = (A = (v'v) * I - 2 * v * v'; norm(A' * A)) -for f in (norm_hermitian1, norm_hermitian2, norm_hermitian3, - norm_hermitian4, norm_hermitian5, norm_hermitian6) +for f in ( + norm_hermitian1, norm_hermitian2, norm_hermitian3, + norm_hermitian4, norm_hermitian5, norm_hermitian6, + ) test_println("VECTOR_TO_NUMBER_FUNCS", f) test_unary_gradient(f, rand(5)) end @@ -222,8 +226,10 @@ view_dot(x) = dot(view(x, 1:3), view(x, 3:5)) view_cartesian(m) = sum(abs2, view(m, 1:2, :)) view_strided(m) = sum(abs2, view(m, :, 2:3)' * view(m, :, 1:2)) -for f in (getindex_logical, getindex_logical_bitvector, view_intermediate, view_and_parent, - view_overlapping, view_nested, view_logical, view_dot) +for f in ( + getindex_logical, getindex_logical_bitvector, view_intermediate, view_and_parent, + view_overlapping, view_nested, view_logical, view_dot, + ) test_println("VECTOR_TO_NUMBER_FUNCS", f) test_unary_gradient(f, rand(5)) end @@ -239,25 +245,25 @@ end @testset "`TrackedReal`" begin g(x) = float(x[1])^3 * x[2] @test ReverseDiff.gradient(g, [2.0, 3.0]) == [36.0, 8.0] - @test ReverseDiff.gradient(g, Rational{Int}[2//1, 3//1]) == [36, 8] + @test ReverseDiff.gradient(g, Rational{Int}[2 // 1, 3 // 1]) == [36, 8] end @testset "`TrackedArray`" begin g(x) = sum((float(x)::ReverseDiff.TrackedArray) .^ 3) @test ReverseDiff.gradient(g, [2.0, 3.0]) == [12.0, 27.0] - @test ReverseDiff.gradient(g, Rational{Int}[2//1, 3//1]) == [12, 27] + @test ReverseDiff.gradient(g, Rational{Int}[2 // 1, 3 // 1]) == [12, 27] end @testset "replaying a recorded tape" begin g(x) = float(x[1])^3 * x[2] - tape = ReverseDiff.GradientTape(g, Rational{Int}[2//1, 3//1]) - @test ReverseDiff.gradient!(tape, Rational{Int}[2//1, 3//1]) == [36, 8] - @test ReverseDiff.gradient!(tape, Rational{Int}[1//1, 4//1]) == [12, 1] + tape = ReverseDiff.GradientTape(g, Rational{Int}[2 // 1, 3 // 1]) + @test ReverseDiff.gradient!(tape, Rational{Int}[2 // 1, 3 // 1]) == [36, 8] + @test ReverseDiff.gradient!(tape, Rational{Int}[1 // 1, 4 // 1]) == [12, 1] ga(x) = sum(float(x) .^ 3) - tape = ReverseDiff.GradientTape(ga, Rational{Int}[2//1, 3//1]) - @test ReverseDiff.gradient!(tape, Rational{Int}[2//1, 3//1]) == [12, 27] - @test ReverseDiff.gradient!(tape, Rational{Int}[1//1, 4//1]) == [3, 48] + tape = ReverseDiff.GradientTape(ga, Rational{Int}[2 // 1, 3 // 1]) + @test ReverseDiff.gradient!(tape, Rational{Int}[2 // 1, 3 // 1]) == [12, 27] + @test ReverseDiff.gradient!(tape, Rational{Int}[1 // 1, 4 // 1]) == [3, 48] end end @@ -403,7 +409,7 @@ end @test ReverseDiff.gradient(x -> sum([Real[x[1], x[2]]; x[3]]), [1.0, 2.0, 3.0]) == [1.0, 1.0, 1.0] @test ReverseDiff.gradient(x -> sum([Any[x[1], x[2]]; x[3]]), [1.0, 2.0, 3.0]) == [1.0, 1.0, 1.0] @test ReverseDiff.gradient(x -> sum([Real[5.0, x[2]]; x[3]]), [1.0, 2.0, 3.0]) == [0.0, 1.0, 1.0] - @test ReverseDiff.gradient(x -> sum([Real[x[1], x[2]] x[3]*ones(2)]), [1.0, 2.0, 3.0]) == [1.0, 1.0, 2.0] + @test ReverseDiff.gradient(x -> sum([Real[x[1], x[2]] x[3] * ones(2)]), [1.0, 2.0, 3.0]) == [1.0, 1.0, 2.0] end ############################################################################################ @@ -434,8 +440,10 @@ end @test ReverseDiff.gradient(x -> sum(sum(mixed(x); dims = 1)), [1.0, 2.0]) == [1.0, 2.0] @test ReverseDiff.gradient(x -> sum(prod(mixed(x); dims = 1)), [3.0, 5.0]) == [10.0, 6.0] - @test ReverseDiff.gradient(x -> sum(sum(reshape(mixed(x), 2, 2); dims = 2)), - [1.0, 2.0, 3.0, 4.0]) == [1.0, 2.0, 1.0, 2.0] + @test ReverseDiff.gradient( + x -> sum(sum(reshape(mixed(x), 2, 2); dims = 2)), + [1.0, 2.0, 3.0, 4.0] + ) == [1.0, 2.0, 1.0, 2.0] @test ReverseDiff.hessian(x -> sum(sum(mixed(x); dims = 1))^2, [1.0, 2.0]) == [2.0 4.0; 4.0 8.0] @@ -459,7 +467,7 @@ end T = size(H, 2) @. est = 0 for lag in 0:(L - 1) - @views mul!(est[:, lag+1:T], W[:, :, lag+1]', H[:, 1:T-lag], 1, 1) + @views mul!(est[:, (lag + 1):T], W[:, :, lag + 1]', H[:, 1:(T - lag)], 1, 1) end return est end diff --git a/test/api/HessianTests.jl b/test/api/HessianTests.jl index c1c49a0a..73d5ffdb 100644 --- a/test/api/HessianTests.jl +++ b/test/api/HessianTests.jl @@ -9,7 +9,7 @@ include(joinpath(dirname(@__FILE__), "../utils.jl")) # out enough of an MRE for the bug to report it... ReverseDiff.hessian(DiffTests.mat2num_1, rand(3, 3)) -hess_test_approx(a, b) = test_approx(a, b, 1e-4) +hess_test_approx(a, b) = test_approx(a, b, 1.0e-4) function test_unary_hessian(f, x) test = DiffResults.HessianResult(x) @@ -80,6 +80,7 @@ function test_unary_hessian(f, x) hess_test_approx(DiffResults.gradient(result), DiffResults.gradient(test)) hess_test_approx(DiffResults.hessian(result), DiffResults.hessian(test)) end + return nothing end for f in DiffTests.MATRIX_TO_NUMBER_FUNCS diff --git a/test/api/JacobianTests.jl b/test/api/JacobianTests.jl index ed3e68c2..eaff84ad 100644 --- a/test/api/JacobianTests.jl +++ b/test/api/JacobianTests.jl @@ -67,6 +67,7 @@ function test_unary_jacobian(f, x) test_approx(DiffResults.value(result), DiffResults.value(test)) test_approx(DiffResults.jacobian(result), DiffResults.jacobian(test)) end + return nothing end function test_unary_jacobian(f!, y, x) @@ -83,7 +84,7 @@ function test_unary_jacobian(f!, y, x) out = similar(DiffResults.jacobian(test)) ReverseDiff.jacobian!(out, f!, y, x) - test_approx(y, DiffResults.value(test)) + test_approx(y, DiffResults.value(test)) test_approx(out, DiffResults.jacobian(test)) copyto!(y, y_original) @@ -99,13 +100,13 @@ function test_unary_jacobian(f!, y, x) cfg = ReverseDiff.JacobianConfig(y, x) out = ReverseDiff.jacobian(f!, y, x, cfg) - test_approx(y, DiffResults.value(test)) + test_approx(y, DiffResults.value(test)) test_approx(out, DiffResults.jacobian(test)) copyto!(y, y_original) out = similar(DiffResults.jacobian(test)) ReverseDiff.jacobian!(out, f!, y, x, cfg) - test_approx(y, DiffResults.value(test)) + test_approx(y, DiffResults.value(test)) test_approx(out, DiffResults.jacobian(test)) copyto!(y, y_original) @@ -150,6 +151,7 @@ function test_unary_jacobian(f!, y, x) test_approx(DiffResults.value(result), DiffResults.value(test)) test_approx(DiffResults.jacobian(result), DiffResults.jacobian(test)) end + return nothing end function test_binary_jacobian(f, a, b) @@ -244,6 +246,7 @@ function test_binary_jacobian(f, a, b) test_approx(DiffResults.gradient(Ja), test_a) test_approx(DiffResults.gradient(Jb), test_b) end + return nothing end for f in (DiffTests.ARRAY_TO_ARRAY_FUNCS..., DiffTests.MATRIX_TO_MATRIX_FUNCS...) diff --git a/test/compat/CompatTests.jl b/test/compat/CompatTests.jl index 9dc08ed8..3b84eb47 100644 --- a/test/compat/CompatTests.jl +++ b/test/compat/CompatTests.jl @@ -3,11 +3,11 @@ module CompatTests using FillArrays, ReverseDiff, Test @test ReverseDiff.gradient(fill(2.0, 3)) do x - sum(abs2.(x .- Zeros(3))) + sum(abs2.(x .- Zeros(3))) end == fill(4.0, 3) @test ReverseDiff.gradient(fill(2.0, 3)) do x - sum(abs2.(x .- (1:3))) + sum(abs2.(x .- (1:3))) end == [2, 0, -2] -end \ No newline at end of file +end diff --git a/test/derivatives/ArrayFunctionTests.jl b/test/derivatives/ArrayFunctionTests.jl index 3bae169b..0012fd6b 100644 --- a/test/derivatives/ArrayFunctionTests.jl +++ b/test/derivatives/ArrayFunctionTests.jl @@ -20,7 +20,7 @@ end # which is ambiguous with the methods defined for `TrackedArray`. const SPARSE_CAT_AMBIGUITY = any(k -> k.name == "SparseArrays", keys(Base.loaded_modules)) -function testcat(f, args::Tuple, type, kwargs=NamedTuple()) +function testcat(f, args::Tuple, type, kwargs = NamedTuple()) x = f(track.(args)...; kwargs...) @test x isa type @test value(x) == f(args...; kwargs...) @@ -65,6 +65,7 @@ function testcat(f, args::Tuple, type, kwargs=NamedTuple()) F = vecx -> sum(f(unpack(sizes, vecx)...; kwargs...)) X = pack(args) @test ForwardDiff.gradient(F, X) == gradient(F, X) + return nothing end function pack(xs) @@ -76,10 +77,10 @@ function unpack(sizes, vecx) start = 0 out = map(sizes) do s if s === () - x = vecx[start+1] + x = vecx[start + 1] start += 1 else - x = reshape(vecx[start+1:start+prod(s)], s) + x = reshape(vecx[(start + 1):(start + prod(s))], s) start += prod(s) end end @@ -88,25 +89,25 @@ end @testset "cat" begin v = rand(3) - m = rand(3,3) - a = rand(3,3,3) + m = rand(3, 3) + a = rand(3, 3, 3) n = rand() - testcat(cat, (n,), TrackedVector, (dims=1,)) - testcat(cat, (n, n), TrackedVector, (dims=1,)) - testcat(cat, (n, n), TrackedMatrix, (dims=2,)) - testcat(cat, (v, n), TrackedVector, (dims=1,)) - testcat(cat, (n, v), TrackedVector, (dims=1,)) - - testcat(cat, (v, v), TrackedVector, (dims=1,)) - testcat(cat, (v, v), TrackedMatrix, (dims=2,)) - testcat(cat, (v, m), TrackedMatrix, (dims=2,)) - testcat(cat, (m, v), TrackedMatrix, (dims=2,)) - - testcat(cat, (a, a), TrackedArray, (dims=1,)) - testcat(cat, (a, a), TrackedArray, (dims=2,)) - testcat(cat, (a, a), TrackedArray, (dims=3,)) - testcat(cat, (a, m), TrackedArray, (dims=3,)) + testcat(cat, (n,), TrackedVector, (dims = 1,)) + testcat(cat, (n, n), TrackedVector, (dims = 1,)) + testcat(cat, (n, n), TrackedMatrix, (dims = 2,)) + testcat(cat, (v, n), TrackedVector, (dims = 1,)) + testcat(cat, (n, v), TrackedVector, (dims = 1,)) + + testcat(cat, (v, v), TrackedVector, (dims = 1,)) + testcat(cat, (v, v), TrackedMatrix, (dims = 2,)) + testcat(cat, (v, m), TrackedMatrix, (dims = 2,)) + testcat(cat, (m, v), TrackedMatrix, (dims = 2,)) + + testcat(cat, (a, a), TrackedArray, (dims = 1,)) + testcat(cat, (a, a), TrackedArray, (dims = 2,)) + testcat(cat, (a, a), TrackedArray, (dims = 3,)) + testcat(cat, (a, m), TrackedArray, (dims = 3,)) testcat(vcat, (n,), TrackedVector) testcat(vcat, (n, n), TrackedVector) diff --git a/test/derivatives/ElementwiseTests.jl b/test/derivatives/ElementwiseTests.jl index 7ce670a1..cdaef6a4 100644 --- a/test/derivatives/ElementwiseTests.jl +++ b/test/derivatives/ElementwiseTests.jl @@ -34,7 +34,7 @@ function test_elementwise(f, fopt, x, tp) # reverse out = similar(y, (length(x), length(x))) ReverseDiff.seeded_reverse_pass!(out, yt, xt, tp) - test_approx(out, ForwardDiff.jacobian(z -> map(f, z), x); nans=true) + test_approx(out, ForwardDiff.jacobian(z -> map(f, z), x); nans = true) # forward x2 = x .- offset @@ -57,7 +57,7 @@ function test_elementwise(f, fopt, x, tp) # reverse out = similar(y, (length(x), length(x))) ReverseDiff.seeded_reverse_pass!(out, yt, xt, tp) - test_approx(out, ForwardDiff.jacobian(z -> broadcast(f, z), x); nans=true) + test_approx(out, ForwardDiff.jacobian(z -> broadcast(f, z), x); nans = true) # forward x2 = x .- offset @@ -67,6 +67,7 @@ function test_elementwise(f, fopt, x, tp) ReverseDiff.value!(xt, x) empty!(tp) + return nothing end function test_map(f, fopt, a, b, tp) @@ -83,7 +84,7 @@ function test_map(f, fopt, a, b, tp) # reverse out = similar(c, (length(c), length(a))) ReverseDiff.seeded_reverse_pass!(out, ct, at, tp) - test_approx(out, ForwardDiff.jacobian(x -> map(f, x, b), a); nans=true) + test_approx(out, ForwardDiff.jacobian(x -> map(f, x, b), a); nans = true) # forward a2 = a .- offset @@ -104,7 +105,7 @@ function test_map(f, fopt, a, b, tp) # reverse out = similar(c, (length(c), length(b))) ReverseDiff.seeded_reverse_pass!(out, ct, bt, tp) - test_approx(out, ForwardDiff.jacobian(x -> map(f, a, x), b); nans=true) + test_approx(out, ForwardDiff.jacobian(x -> map(f, a, x), b); nans = true) # forward b2 = b .- offset @@ -127,13 +128,13 @@ function test_map(f, fopt, a, b, tp) out_b = similar(c, (length(c), length(b))) ReverseDiff.seeded_reverse_pass!(out_a, ct, at, tp) ReverseDiff.seeded_reverse_pass!(out_b, ct, bt, tp) - jac = let a=a, b=b, f=f + jac = let a = a, b = b, f = f ForwardDiff.jacobian(vcat(vec(a), vec(b))) do x map(f, reshape(x[1:length(a)], size(a)), reshape(x[(length(a) + 1):end], size(b))) end end - test_approx(out_a, jac[:, 1:length(a)]; nans=true) - test_approx(out_b, jac[:, (length(a) + 1):end]; nans=true) + test_approx(out_a, jac[:, 1:length(a)]; nans = true) + test_approx(out_b, jac[:, (length(a) + 1):end]; nans = true) # forward a2, b2 = a .- offset, b .- offset ReverseDiff.value!(at, a2) @@ -144,6 +145,7 @@ function test_map(f, fopt, a, b, tp) ReverseDiff.value!(bt, b) empty!(tp) + return nothing end function test_broadcast(f, fopt, a::AbstractArray, b::AbstractArray, tp, builtin::Bool = false) @@ -167,7 +169,7 @@ function test_broadcast(f, fopt, a::AbstractArray, b::AbstractArray, tp, builtin # reverse out = similar(c, (length(c), length(a))) ReverseDiff.seeded_reverse_pass!(out, ct, at, tp) - test_approx(out, ForwardDiff.jacobian(x -> g(x, b), a); nans=true) + test_approx(out, ForwardDiff.jacobian(x -> g(x, b), a); nans = true) # forward a2 = a .- offset @@ -188,7 +190,7 @@ function test_broadcast(f, fopt, a::AbstractArray, b::AbstractArray, tp, builtin # reverse out = similar(c, (length(c), length(b))) ReverseDiff.seeded_reverse_pass!(out, ct, bt, tp) - test_approx(out, ForwardDiff.jacobian(x -> g(a, x), b); nans=true) + test_approx(out, ForwardDiff.jacobian(x -> g(a, x), b); nans = true) # forward b2 = b .- offset @@ -211,13 +213,13 @@ function test_broadcast(f, fopt, a::AbstractArray, b::AbstractArray, tp, builtin out_b = similar(c, (length(c), length(b))) ReverseDiff.seeded_reverse_pass!(out_a, ct, at, tp) ReverseDiff.seeded_reverse_pass!(out_b, ct, bt, tp) - jac = let a=a, b=b, g=g + jac = let a = a, b = b, g = g ForwardDiff.jacobian(vcat(vec(a), vec(b))) do x g(reshape(x[1:length(a)], size(a)), reshape(x[(length(a) + 1):end], size(b))) end end - test_approx(out_a, jac[:, 1:length(a)]; nans=true) - test_approx(out_b, jac[:, (length(a) + 1):end]; nans=true) + test_approx(out_a, jac[:, 1:length(a)]; nans = true) + test_approx(out_b, jac[:, (length(a) + 1):end]; nans = true) # forward a2, b2 = a .- offset, b .- offset @@ -229,6 +231,7 @@ function test_broadcast(f, fopt, a::AbstractArray, b::AbstractArray, tp, builtin ReverseDiff.value!(bt, b) empty!(tp) + return nothing end function test_broadcast(f, fopt, n::Number, x::AbstractArray, tp, builtin::Bool = false) @@ -252,7 +255,7 @@ function test_broadcast(f, fopt, n::Number, x::AbstractArray, tp, builtin::Bool # reverse out = similar(y) ReverseDiff.seeded_reverse_pass!(out, yt, nt, tp) - test_approx(out, ForwardDiff.derivative(z -> g(z, x), n); nans=true) + test_approx(out, ForwardDiff.derivative(z -> g(z, x), n); nans = true) # forward n2 = n + offset @@ -273,7 +276,7 @@ function test_broadcast(f, fopt, n::Number, x::AbstractArray, tp, builtin::Bool # reverse out = similar(y, (length(y), length(x))) ReverseDiff.seeded_reverse_pass!(out, yt, xt, tp) - test_approx(out, ForwardDiff.jacobian(z -> g(n, z), x); nans=true) + test_approx(out, ForwardDiff.jacobian(z -> g(n, z), x); nans = true) # forward x2 = x .- offset @@ -296,14 +299,14 @@ function test_broadcast(f, fopt, n::Number, x::AbstractArray, tp, builtin::Bool out_x = similar(y, (length(y), length(x))) ReverseDiff.seeded_reverse_pass!(out_n, yt, nt, tp) ReverseDiff.seeded_reverse_pass!(out_x, yt, xt, tp) - jac = let x=x, g=g + jac = let x = x, g = g ForwardDiff.jacobian(z -> g(z[1], reshape(z[2:end], size(x))), vcat(n, vec(x))) end - test_approx(out_n, reshape(jac[:, 1], size(y)); nans=true) - test_approx(out_x, jac[:, 2:end]; nans=true) + test_approx(out_n, reshape(jac[:, 1], size(y)); nans = true) + test_approx(out_x, jac[:, 2:end]; nans = true) # forward - n2, x2 = n + offset , x .- offset + n2, x2 = n + offset, x .- offset ReverseDiff.value!(nt, n2) ReverseDiff.value!(xt, x2) ReverseDiff.forward_pass!(tp) @@ -312,6 +315,7 @@ function test_broadcast(f, fopt, n::Number, x::AbstractArray, tp, builtin::Bool ReverseDiff.value!(xt, x) empty!(tp) + return nothing end function test_broadcast(f, fopt, x::AbstractArray, n::Number, tp, builtin::Bool = false) @@ -335,7 +339,7 @@ function test_broadcast(f, fopt, x::AbstractArray, n::Number, tp, builtin::Bool # reverse out = similar(y) ReverseDiff.seeded_reverse_pass!(out, yt, nt, tp) - test_approx(out, ForwardDiff.derivative(z -> g(x, z), n); nans=true) + test_approx(out, ForwardDiff.derivative(z -> g(x, z), n); nans = true) # forward n2 = n + offset @@ -356,7 +360,7 @@ function test_broadcast(f, fopt, x::AbstractArray, n::Number, tp, builtin::Bool # reverse out = similar(y, (length(y), length(x))) ReverseDiff.seeded_reverse_pass!(out, yt, xt, tp) - test_approx(out, ForwardDiff.jacobian(z -> g(z, n), x); nans=true) + test_approx(out, ForwardDiff.jacobian(z -> g(z, n), x); nans = true) # forward x2 = x .- offset @@ -379,11 +383,11 @@ function test_broadcast(f, fopt, x::AbstractArray, n::Number, tp, builtin::Bool out_x = similar(y, (length(y), length(x))) ReverseDiff.seeded_reverse_pass!(out_n, yt, nt, tp) ReverseDiff.seeded_reverse_pass!(out_x, yt, xt, tp) - jac = let x=x, g=g + jac = let x = x, g = g ForwardDiff.jacobian(z -> g(reshape(z[1:(end - 1)], size(x)), z[end]), vcat(vec(x), n)) end - test_approx(out_x, jac[:, 1:(end - 1)]; nans=true) - test_approx(out_n, reshape(jac[:, end], size(y)); nans=true) + test_approx(out_x, jac[:, 1:(end - 1)]; nans = true) + test_approx(out_n, reshape(jac[:, end], size(y)); nans = true) # forward x2, n2 = x .- offset, n + offset @@ -395,6 +399,7 @@ function test_broadcast(f, fopt, x::AbstractArray, n::Number, tp, builtin::Bool ReverseDiff.value!(nt, n) empty!(tp) + return nothing end for f in DiffTests.NUMBER_TO_NUMBER_FUNCS @@ -403,7 +408,7 @@ for f in DiffTests.NUMBER_TO_NUMBER_FUNCS test_elementwise(f, ReverseDiff.@forward(f), a, tp) end -for (M, fsym, arity) in DiffRules.diffrules(; filter_modules=nothing) +for (M, fsym, arity) in DiffRules.diffrules(; filter_modules = nothing) # ensure that all rules can be tested if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), fsym)) error("$M.$fsym is not available") @@ -444,7 +449,7 @@ for f in DiffTests.BINARY_BROADCAST_OPS end @testset "broadcasting over types without a definite field count (#289)" begin - T = ReverseDiff.TrackedReal{Float64,Float64,ReverseDiff.TrackedArray{Float64,Float64,1,Vector{Float64},Vector{Float64}}} + T = ReverseDiff.TrackedReal{Float64, Float64, ReverseDiff.TrackedArray{Float64, Float64, 1, Vector{Float64}, Vector{Float64}}} msg = "Converting an instance of $T to Float64 is not defined. Please use `ReverseDiff.value` instead." @test_throws ArgumentError(msg) ReverseDiff.gradient(v -> sum(convert.(Float64, v) .* [2.0, 3.0]), [0.3, 0.5]) diff --git a/test/derivatives/LinAlgTests.jl b/test/derivatives/LinAlgTests.jl index 59bbb833..6b961c58 100644 --- a/test/derivatives/LinAlgTests.jl +++ b/test/derivatives/LinAlgTests.jl @@ -40,6 +40,7 @@ function test_arr2num(f, x, tp; ignore_tape_length = false) ReverseDiff.value!(xt, x) empty!(tp) + return nothing end function test_arr2arr(f, x, tp) @@ -64,6 +65,7 @@ function test_arr2arr(f, x, tp) ReverseDiff.value!(xt, x) empty!(tp) + return nothing end function test_arr2arr(f, a, b, tp) @@ -137,6 +139,7 @@ function test_arr2arr(f, a, b, tp) ReverseDiff.value!(bt, b) empty!(tp) + return nothing end function test_arr2arr_inplace(f!, f, c, a, b, tp) @@ -213,20 +216,21 @@ function test_arr2arr_inplace(f!, f, c, a, b, tp) ReverseDiff.value!(bt, b) empty!(tp) + return nothing end for f in ( - sum, - det, - mean, - y -> dot(vec(y), vec(y)), - y -> vec(y)' * vec(y), - y -> transpose(vec(y)) * vec(y), - y -> vec(y)' * ones(length(y)), - y -> transpose(vec(y)) * ones(length(y)), - y -> ones(length(y))' * vec(y), - y -> transpose(ones(length(y))) * vec(y), -) + sum, + det, + mean, + y -> dot(vec(y), vec(y)), + y -> vec(y)' * vec(y), + y -> transpose(vec(y)) * vec(y), + y -> vec(y)' * ones(length(y)), + y -> transpose(vec(y)) * ones(length(y)), + y -> ones(length(y))' * vec(y), + y -> transpose(ones(length(y))) * vec(y), + ) test_println("Array -> Number functions", f) test_arr2num(f, x, tp) end @@ -238,11 +242,11 @@ function norm_hermitian(v) end for f in ( - y -> vec(y)' * Matrix{Float64}(I, length(y), length(y)) * vec(y), - norm_hermitian, -) + y -> vec(y)' * Matrix{Float64}(I, length(y), length(y)) * vec(y), + norm_hermitian, + ) test_println("Array -> Number functions", f) - test_arr2num(f, x, tp, ignore_tape_length=true) + test_arr2num(f, x, tp, ignore_tape_length = true) end for f in (-, inv) @@ -287,7 +291,7 @@ test_arr2arr_inplace(mul!, *, x, transpose(a), adjoint(b), tp) # Prevent regression on https://github.com/JuliaDiff/ReverseDiff.jl/issues/235 A = [1 2; 3 4]; x = [5, 6]; -@test ReverseDiff.gradient(y -> sum(y'*A), x) == [3, 7] +@test ReverseDiff.gradient(y -> sum(y' * A), x) == [3, 7] # Derivatives w.r.t. structured factors (https://github.com/JuliaDiff/DiffTests.jl/issues/13) x = rand(3, 3) @@ -297,12 +301,12 @@ mask(S) = Diagonal(vec(Matrix(S(ones(3, 3))))) jac_lmul(S) = kron(transpose(x), I(3)) * mask(S) + kron(I(3), Matrix(S(x))) jac_rmul(S) = kron(transpose(Matrix(S(x))), I(3)) + kron(I(3), x) * mask(S) for (f, J) in ( - (y -> Diagonal(y) * y, jac_lmul(Diagonal)), - (y -> UpperTriangular(y) * y, jac_lmul(UpperTriangular)), - (y -> y * LowerTriangular(y), jac_rmul(LowerTriangular)), - (y -> transpose(UpperTriangular(transpose(y))) * y, jac_lmul(LowerTriangular)), - (y -> y * adjoint(LowerTriangular(adjoint(y))), jac_rmul(UpperTriangular)), -) + (y -> Diagonal(y) * y, jac_lmul(Diagonal)), + (y -> UpperTriangular(y) * y, jac_lmul(UpperTriangular)), + (y -> y * LowerTriangular(y), jac_rmul(LowerTriangular)), + (y -> transpose(UpperTriangular(transpose(y))) * y, jac_lmul(LowerTriangular)), + (y -> y * adjoint(LowerTriangular(adjoint(y))), jac_rmul(UpperTriangular)), + ) test_println("*(A, B) functions with structured matrices", f) test_approx(ReverseDiff.jacobian(f, x), J) ctp = ReverseDiff.compile(ReverseDiff.JacobianTape(f, x)) @@ -313,6 +317,7 @@ function test_jacobian(f, x, J) test_approx(ReverseDiff.jacobian(f, x), J) ctp = ReverseDiff.compile(ReverseDiff.JacobianTape(f, x)) test_approx(ReverseDiff.jacobian!(ctp, x), J) + return nothing end # Constant structured factors @@ -329,12 +334,12 @@ v, V, M = rand(3), rand(2, 1), rand(3, 2) for f in (transpose, adjoint) test_println("*(A, B) functions with row vectors", f) for (h, x) in ( - ((f, u) -> u * f(u), rand(3)), - ((f, u) -> u * f(v), rand(2, 1)), - ((f, u) -> V * f(u), rand(3)), - ((f, u) -> f(u) * M, rand(3)), - ((f, u) -> f(v) * u, rand(3, 2)), - ) + ((f, u) -> u * f(u), rand(3)), + ((f, u) -> u * f(v), rand(2, 1)), + ((f, u) -> V * f(u), rand(3)), + ((f, u) -> f(u) * M, rand(3)), + ((f, u) -> f(v) * u, rand(3, 2)), + ) test_jacobian(u -> h(f, u), x, ReverseDiff.jacobian(u -> h(x -> reshape(x, 1, :), u), x)) end end @@ -345,11 +350,11 @@ end diagreal(u) = Real[u[10], u[11], u[12]] E = Matrix(I(9))[:, [1, 5, 9]] for (f, J) in ( - (u -> dense(u) + Diagonal(diagreal(u)), [I(9) E]), - (u -> dense(u) - diagm(diagreal(u)), [I(9) -E]), - (u -> diagm(diagreal(u)) - dense(u), [-I(9) E]), - (u -> dense(u) + Real[i == j ? u[9 + i] : 0.0 for i in 1:3, j in 1:3], [I(9) E]), - ) + (u -> dense(u) + Diagonal(diagreal(u)), [I(9) E]), + (u -> dense(u) - diagm(diagreal(u)), [I(9) -E]), + (u -> diagm(diagreal(u)) - dense(u), [-I(9) E]), + (u -> dense(u) + Real[i == j ? u[9 + i] : 0.0 for i in 1:3, j in 1:3], [I(9) E]), + ) test_jacobian(f, rand(12), J) end end @@ -358,46 +363,46 @@ end x = rand(2) X = rand(2, 2) for (sv, sm) in ( - (SVector(1.0, 2.0), SMatrix{2,2}(1.0, 2.0, 3.0, 4.0)), - (MVector(1.0, 2.0), MMatrix{2,2}(1.0, 2.0, 3.0, 4.0)), - (SizedVector{2}([1.0, 2.0]), SizedMatrix{2,2}([1.0 3.0; 2.0 4.0])), - (SVector{2,BigFloat}(1, 2), SMatrix{2,2,BigFloat}(1, 2, 3, 4)), - ) - for (f, J) in ( - (u -> sv + u, I(2)), - (u -> u + sv, I(2)), - (u -> sv - u, -I(2)), - (u -> u - sv, I(2)), - (u -> sm * u, sm), + (SVector(1.0, 2.0), SMatrix{2, 2}(1.0, 2.0, 3.0, 4.0)), + (MVector(1.0, 2.0), MMatrix{2, 2}(1.0, 2.0, 3.0, 4.0)), + (SizedVector{2}([1.0, 2.0]), SizedMatrix{2, 2}([1.0 3.0; 2.0 4.0])), + (SVector{2, BigFloat}(1, 2), SMatrix{2, 2, BigFloat}(1, 2, 3, 4)), ) + for (f, J) in ( + (u -> sv + u, I(2)), + (u -> u + sv, I(2)), + (u -> sv - u, -I(2)), + (u -> u - sv, I(2)), + (u -> sm * u, sm), + ) test_jacobian(f, x, J) end for (f, J) in ( - (U -> sm + U, I(4)), - (U -> U + sm, I(4)), - (U -> sm - U, -I(4)), - (U -> U - sm, I(4)), - (U -> sm * U, kron(I(2), Matrix(sm))), - (U -> U * sm, kron(transpose(Matrix(sm)), I(2))), - (U -> U * sv, kron(transpose(Vector(sv)), I(2))), - ) + (U -> sm + U, I(4)), + (U -> U + sm, I(4)), + (U -> sm - U, -I(4)), + (U -> U - sm, I(4)), + (U -> sm * U, kron(I(2), Matrix(sm))), + (U -> U * sm, kron(transpose(Matrix(sm)), I(2))), + (U -> U * sv, kron(transpose(Vector(sv)), I(2))), + ) test_jacobian(f, X, J) end # record at another input so that replaying must recompute the static values for (f, g) in ( - (u -> sum(abs2, sm * u + sv), u -> 2 * sm' * (sm * u + sv)), - (u -> sum(abs2, u' * sm), u -> 2 * sm * sm' * u), - (u -> sum(abs2, transpose(u) * sm), u -> 2 * sm * transpose(sm) * u), - ) + (u -> sum(abs2, sm * u + sv), u -> 2 * sm' * (sm * u + sv)), + (u -> sum(abs2, u' * sm), u -> 2 * sm * sm' * u), + (u -> sum(abs2, transpose(u) * sm), u -> 2 * sm * transpose(sm) * u), + ) tp = ReverseDiff.GradientTape(f, rand(2)) test_approx(ReverseDiff.gradient!(tp, x), g(x)) end end for (f, J) in ( - (u -> SVector(u[1], u[2]) + u, 2I(2)), - (u -> u - SVector(u[1], u[2]), zeros(2, 2)), - (u -> SMatrix{2,2}(u[1], u[2], u[1], u[2]) * u, [2x[1]+x[2] x[1]; x[2] x[1]+2x[2]]), - ) + (u -> SVector(u[1], u[2]) + u, 2I(2)), + (u -> u - SVector(u[1], u[2]), zeros(2, 2)), + (u -> SMatrix{2, 2}(u[1], u[2], u[1], u[2]) * u, [2x[1] + x[2] x[1]; x[2] x[1] + 2x[2]]), + ) test_jacobian(f, x, J) tp = ReverseDiff.JacobianTape(f, rand(2)) test_approx(ReverseDiff.jacobian!(tp, x), J) diff --git a/test/derivatives/ScalarTests.jl b/test/derivatives/ScalarTests.jl index 04f28d83..3bb2ae60 100644 --- a/test/derivatives/ScalarTests.jl +++ b/test/derivatives/ScalarTests.jl @@ -38,6 +38,7 @@ function test_forward(f, x, tp::InstructionTape, fsym::Symbol) ReverseDiff.value!(xt, x) empty!(tp) + return nothing end function test_forward(f, a, b, tp) @@ -55,7 +56,7 @@ function test_forward(f, a, b, tp) # reverse ReverseDiff.seed!(ct) ReverseDiff.reverse_pass!(tp) - test_approx(deriv(at), ForwardDiff.derivative(x -> f(x, b), a); nans=true) + test_approx(deriv(at), ForwardDiff.derivative(x -> f(x, b), a); nans = true) ReverseDiff.unseed!(at) # forward @@ -77,7 +78,7 @@ function test_forward(f, a, b, tp) # reverse ReverseDiff.seed!(ct) ReverseDiff.reverse_pass!(tp) - test_approx(deriv(bt), ForwardDiff.derivative(x -> f(a, x), b); nans=true) + test_approx(deriv(bt), ForwardDiff.derivative(x -> f(a, x), b); nans = true) ReverseDiff.unseed!(bt) # forward @@ -100,8 +101,8 @@ function test_forward(f, a, b, tp) ReverseDiff.seed!(ct) ReverseDiff.reverse_pass!(tp) grads = ForwardDiff.gradient(x -> f(x[1], x[2]), [a, b]) - test_approx(deriv(at), grads[1]; nans=true) - test_approx(deriv(bt), grads[2]; nans=true) + test_approx(deriv(at), grads[1]; nans = true) + test_approx(deriv(bt), grads[2]; nans = true) # forward a2 = isa(a, Int) ? rand(int_range) : rand() @@ -114,6 +115,7 @@ function test_forward(f, a, b, tp) ReverseDiff.value!(bt, b) empty!(tp) + return nothing end function test_skip(f, x, tp) @@ -122,6 +124,7 @@ function test_skip(f, x, tp) yt = f(xt) @test yt == y @test isempty(tp) + return nothing end function test_skip(f, a, b, tp) @@ -140,9 +143,10 @@ function test_skip(f, a, b, tp) ct = f(at, bt) @test ct == c @test isempty(tp) + return nothing end -for (M, f, arity) in DiffRules.diffrules(; filter_modules=nothing) +for (M, f, arity) in DiffRules.diffrules(; filter_modules = nothing) # ensure that function is defined if !(isdefined(@__MODULE__, M) && isdefined(getfield(@__MODULE__, M), f)) error("$M.$f is not available") diff --git a/test/runtests.jl b/test/runtests.jl index b53f0e80..4f595706 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,7 +2,7 @@ using Test const TESTDIR = dirname(@__FILE__) -test_println(kind, f, pad=" ") = println(pad, "testing $(kind): `$(f)`...") +test_println(kind, f, pad = " ") = println(pad, "testing $(kind): `$(f)`...") @testset "ReverseDiff" begin diff --git a/test/utils.jl b/test/utils.jl index de11fa22..c21e491e 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,6 @@ using ReverseDiff: InstructionTape, GradientConfig, JacobianConfig, HessianConfig, - value, deriv, tape, valtype, - derivtype, track, track! + value, deriv, tape, valtype, + derivtype, track, track! using Random @@ -15,7 +15,7 @@ Random.seed!(1) test_println(kind, f, pad = " ") = println(pad, "testing $(kind): `$(f)`...") -@inline test_approx(A, B, _atol = 1e-5; nans::Bool=false) = @test isapprox(A, B; atol = _atol, nans=nans) +@inline test_approx(A, B, _atol = 1.0e-5; nans::Bool = false) = @test isapprox(A, B; atol = _atol, nans = nans) tracked_is(a, b) = value(a) === value(b) && deriv(a) === deriv(b) && tape(a) === tape(b) tracked_is(a::AbstractArray, b::AbstractArray) = all(map(tracked_is, a, b))