-
Notifications
You must be signed in to change notification settings - Fork 34
Expand file tree
/
Copy pathutils.jl
More file actions
1860 lines (1592 loc) · 70.8 KB
/
utils.jl
File metadata and controls
1860 lines (1592 loc) · 70.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Collect generic utility functions in this file
""" Return the decimal part of a float number `x`"""
getdecimal(x::Number) = x - trunc(Int, x)
""" Return an ierator over data skipping non-finite values"""
skipnan(itr) = Iterators.filter(el->isfinite(el), itr)
pow(b,e) = b ^ e
"""
funcurve(f::Function, lims, n=100)::Tuple{Vector{Float64}, Vector{Float64}}
Evaluate a function over a range and return the computed points.
# Arguments
- `f::Function`: The one dimensional function to be evaluated. This can be the named of a built-in function
(_e.g._, `exp`), a custom or an anonymous function.
- `lims`: The limits defining the range over which to evaluate the function.
That is, the limits of `x` in `y = f(x)`. This can be a tuple or a vector with two elements.
- `n::Int`: Number of points to evaluate (default: 100)
# Returns
- `Tuple{Vector{Float64}, Vector{Float64}}`: A tuple containing two vectors of Float64 values
representing the x-coordinates and corresponding function values
# Example
```julia
x, y = funcurve(exp, [0, 10])
```
Another example with a custom function (an anonymous function in this case):
```julia
x, y = funcurve(x->x^2, (0, 10))
```
Visualize the result with:
```julia
viz(x,y)
```
"""
function funcurve(f::Function, lims, n::Integer=100)::Tuple{Vector{Float64}, Vector{Float64}}
x = collect(linspace(lims[1], lims[2], n))
y = f.(x)
return x, y
end
#=
"""
"""
function autocorr(x; demean=true)
N = length(x)
a = zeros(N)
x1 = vec(x) .- mean(x)
x2 = deepcopy(x1)
a[1] = sum(x1 .* x1)
for k = 2:N
circshift!(x2,-1)
t = view(x2, 1:(N-k+1))
a[k] = sum(t .* view(x1, 1:(N-k+1)))
end
a ./= a[1]
end
=#
"""
x, y = pol2cart(theta, rho; deg=false)
Transform polar to Cartesian coordinates. Angles are in radians by default.
Use `deg=true` if angles are in degrees. Input can be scalar, vectors or matrices.
"""
function pol2cart(theta, rho; deg::Bool=false)
return (deg) ? (rho .* cosd.(theta), rho .* sind.(theta)) : (rho .* cos.(theta), rho .* sin.(theta))
end
"""
theta, rho = cart2pol(x, y; deg=false)
Transform Cartesian to polar coordinates. Angles are returned in radians by default.
Use `deg=true` to return the angles in degrees. Input can be scalar, vectors or matrices.
"""
cart2pol(x, y; deg::Bool=false) = (deg) ? atand.(y,x) : atan.(y,x), hypot.(x,y)
"""
x, y, z = sph2cart(az, elev, rho; deg=false)
Transform spherical coordinates to Cartesian. Angles are in radians by default.
Use `deg=true` if angles are in degrees. Input can be scalar, vectors or matrices.
"""
function sph2cart(az, elev, rho; deg::Bool=false)
z = rho .* ((deg) ? sind.(elev) : sin.(elev))
t = rho .* ((deg) ? cosd.(elev) : cos.(elev))
x = t .* ((deg) ? cosd.(az) : cos.(az))
y = t .* ((deg) ? sind.(az) : sin.(az))
return x,y,z
end
"""
az, elev, rho = cart2sph(x, y, z; deg=false)
Transform Cartesian coordinates to spherical. Angles are returned in radians by default.
Use `deg=true` to return the angles in degrees. Input can be scalar, vectors or matrices.
"""
function cart2sph(x, y, z; deg::Bool=false)
h = hypot.(x, y)
rho = hypot.(h, z)
elev = (deg) ? atand.(z, h) : atan.(z, h)
az = (deg) ? atand.(y, x) : atan.(y, x)
return az, elev, rho
end
# ----------------------------------------------------------------------------------------------------------
"""
count_chars(str::AbstractString, c::Char[=','])
Count the number of characters `c` in the AbstracString `str`
"""
function count_chars(str::AbstractString, c::Char=',')::Int
count(i->(i == c), str)
end
# ----------------------------------------------------------------------------------------------------------
"""
ind = ind2bool(x::Vector{<:Integer} [, maxind::Int=0]) -> BitVector
Convert a vector of indices `x` to a boolean vector `ind` where positions in `x` are true.
If `maxind` is provided, the output boolean vector will have length `maxind`, otherwise its length will be `maximum(x)`.
"""
function ind2bool(indices::Vector{<:Integer}, maxind::Int=0)::BitVector
bool_vec = (maxind > 0) ? falses(maxind) : falses(maximum(indices))
bool_vec[indices] .= true
return bool_vec
end
# ----------------------------------------------------------------------------------------------------------
"""
ind = uniqueind(x)
Return the index `ind` such that x[ind] gets the unique values of x. No sorting is done
"""
uniqueind(x) = unique(i -> x[i], eachindex(x))
"""
u, ind = gunique(x::AbstractVector; sorted=false)
Return an array containing only the unique elements of `x` and the indices `ind` such that `u = x[ind]`.
If `sorted` is true the output is sorted (default is not)
u, ic, ia = gunique(x::AbstractMatrix; sorted::Bool=false, rows=true)
Behaves like Matlab's unique(x, 'rows'), where u = x(ia,:) and x = u(ic,:). If `rows` is false then `ic` is empty.
"""
function gunique(x::AbstractVector; sorted::Bool=false)
function uniquit(x)
if sorted
_ind = sortperm(x)
_x = x[_ind]
ind = uniqueind(_x)
return _x[ind], _ind[ind]
else
ind = uniqueind(x)
return x[ind], ind
end
end
if (eltype(x) <: AbstractFloat && any(isnan.(x)))
uniquit(collect(skipnan(x)))
else
uniquit(x)
end
end
# Method for matrices. The default is like Matalb's [C, ind] = unique(A, 'rows')
function gunique(x::AbstractMatrix; sorted::Bool=false, rows=true)
!rows && return gunique(vec(x); sorted=sorted)
if (sorted)
#xx = view(x, ind,:)
#ind_ = sortslicesperm(xx, dims=1)
#ind = ind[ind_]
rows = size(x,1)
IC = sortslicesperm(x, dims=1)
C = x[IC, :]
d = view(C, 1:rows-1, :) .!= view(C, 2:rows, :)
d = any(d, dims=2)
d = vec([true; d]) # First row is always a member of unique list.
C = C[d, :]
IA = cumsum(d, dims=1) # Lists position, starting at 1.
IA[IC] = IA #Re-reference POS to indexing of SORT.
IC = IC[d]
return C, IC, IA
end
ind = first.(unique(last,pairs(eachrow(x))))
return x[ind, :], ind, Vector{Int}()
end
# ----------------------------------------------------------------------------------------------------------
"""
p = sortslicesperm(A; dims=1, kws...)
Like `sortslices` but return instead the indices `p` such that `A[p, :] == sortslices(A; dims=1, kws...)`
"""
function sortslicesperm(A::AbstractArray; dims::Union{Integer, Tuple{Vararg{Integer}}}=1, kws...)
itspace = compute_itspace(A, Val{dims}())
vecs = map(its->view(A, its...), itspace)
sortperm(vecs; kws...)
end
# Works around inference's lack of ability to recognize partial constness
struct DimSelector{dims, T}
A::T
end
DimSelector{dims}(x::T) where {dims, T} = DimSelector{dims, T}(x)
(ds::DimSelector{dims, T})(i) where {dims, T} = i in dims ? axes(ds.A, i) : (:,)
_negdims(n, dims) = filter(i->!(i in dims), 1:n)
function compute_itspace(A, ::Val{dims}) where {dims}
negdims = _negdims(ndims(A), dims)
axs = Iterators.product(ntuple(DimSelector{dims}(A), ndims(A))...)
vec(permutedims(collect(axs), (dims..., negdims...)))
end
#= ------------------------------------------------------------
"""
Delete rows from a matrix where any of its columns has a NaN
"""
function del_mat_row_nans(mat)
nanrows = any(isnan.(mat), dims=2)
return (any(nanrows)) ? mat[vec(.!nanrows),:] : mat
end
=#
# ----------------------------------------------------------------------------------------------------------
function std_nan(A, dims=1)
# Compute std of a matrix or vector accounting for the possibility of NaNs.
if (any(isnan.(A)))
N = (dims == 1) ? size(A, 2) : size(A, 1)
S = zeros(1, N)
if (dims == 1)
for k = 1:N S[k] = std(skipnan(view(A, :,k))) end
else
for k = 1:N S[k] = std(skipnan(view(A, k,:))) end
end
else
S = std(A, dims=dims)
end
end
# --------------------------------------------------------------------------------------------------
# Incredibly Julia ignores the NaN nature and incredibly min(1,NaN) = NaN, so need to ... fck
extrema_nan(A::AbstractArray{<:AbstractFloat}) = minimum_nan(A), maximum_nan(A)
extrema_nodataval(A::AbstractArray{<:Real}, val) = minimum_nodataval(A, val), maximum_nodataval(A, val)
extrema_nan(A) = extrema(A)
"""
extrema_cols(A; col=1)
Compute the minimum and maximum of a column of a matrix/vector `A` ignoring NaNs
"""
function extrema_cols(A; col=1)
(col > size(A,2)) && error("'col' ($col) larger than number of coluns in array ($(size(A,2)))")
mi, ma = typemax(eltype(A)), typemin(eltype(A))
@inbounds for n = 1:size(A,1)
mi = ifelse(mi > A[n,col], A[n,col], mi)
ma = ifelse(ma < A[n,col], A[n,col], ma)
end
return mi, ma
end
"""
extrema_cols_nan(A; col=1)
Compute the minimum and maximum of a column of a matrix/vector `A` NOT ignoring NaNs or Infs
"""
function extrema_cols_nan(A; col=1)
(col > size(A,2)) && error("'col' ($col) larger than number of coluns in array ($(size(A,2)))")
mi, ma = typemax(eltype(A)), typemin(eltype(A))
@inbounds for n = 1:size(A,1)
if isfinite(A[n,col])
mi = ifelse(mi > A[n,col], A[n,col], mi)
ma = ifelse(ma < A[n,col], A[n,col], ma)
end
end
return mi, ma
end
function minimum_nan(A::AbstractArray{<:AbstractFloat})
mi = minimum(A); !isnan(mi) && return mi # The noNaNs version is a order of magnitude faster
mi = typemax(eltype(A))
@inbounds for k in eachindex(A) mi = ifelse(isfinite(A[k]), min(mi, A[k]), mi) end
mi == typemax(eltype(A)) && (mi = convert(eltype(A), NaN)) # Better to return NaN then +Inf
return mi
end
minimum_nan(A) = minimum(A)
function maximum_nan(A::AbstractArray{<:AbstractFloat})
ma = maximum(A); !isnan(ma) && return ma # The noNaNs version is a order of magnitude faster
ma = typemin(eltype(A))
@inbounds for k in eachindex(A) ma = ifelse(isfinite(A[k]), max(ma, A[k]), ma) end
ma == typemin(eltype(A)) && (ma = convert(eltype(A), NaN)) # Better to return NaN than -Inf
return ma
end
maximum_nan(A) = maximum(A)
# Cases where a specific no-data value is used and therefore should be excluded from min/max calculations
function minimum_nodataval(A::AbstractArray{<:Real}, val)
(val == typemax(eltype(A))) && return minimum(A) # No need to check for no-data value
mi = typemax(eltype(A))
@inbounds for k in eachindex(A) mi = ifelse((A[k] != val), min(mi, A[k]), mi) end
return mi
end
function maximum_nodataval(A::AbstractArray{<:Real}, val)
(val == typemin(eltype(A))) && return maximum(A) # No need to check for no-data value
ma = typemin(eltype(A))
@inbounds for k in eachindex(A) ma = ifelse((A[k] != val), max(ma, A[k]), ma) end
return ma
end
function findmax_nan(x::AbstractVector{T}) where T
# Since Julia doesn't ignore NaNs and prefer to return wrong results findmax is useless when data
# has NaNs. We start by runing findmax() and only if max is NaN we fallback to a slower algorithm.
ma, ind = findmax(x)
if (isnan(ma))
ma, ind = typemin(eltype(x)), 0
for k in eachindex(x)
!isnan(x[k]) && ((x[k] > ma) && (ma = x[k]; ind = k))
end
end
ma, ind
end
function findmin_nan(x::AbstractVector{T}) where T
mi, ind = findmin(x)
if (isnan(mi))
mi, ind = typemax(eltype(x)), 0
for k in eachindex(x)
!isnan(x[k]) && ((x[k] < mi) && (mi = x[k]; ind = k))
end
end
mi, ind
end
nanmean(x) = mean(filter(!isnan,x))
nanmean(x,y) = mapslices(nanmean,x,dims=y)
nanstd(x) = std(filter(!isnan,x))
nanstd(x,y) = mapslices(nanstd,x,dims=y)
# --------------------------------------------------------------------------------------------------
Base.minimum(A::Array{<:Complex{<:Integer}}) = minimum(real(A)), minimum(imag(A))
Base.maximum(A::Array{<:Complex{<:Integer}}) = maximum(real(A)), maximum(imag(A))
Base.minimum(A::Array{<:Complex{<:AbstractFloat}}) = minimum(real(A)), minimum(imag(A))
Base.maximum(A::Array{<:Complex{<:AbstractFloat}}) = maximum(real(A)), maximum(imag(A))
function Base.extrema(A::Array{<:Complex{<:Integer}}) # Returns real_min, real_max, imag_min, imag_max
mi_r, mi_i = minimum(A), maximum(A)
return mi_r[1], mi_i[1], mi_r[2], mi_i[2]
end
function Base.extrema(A::Array{<:Complex{<:Real}})
mi_r, mi_i = minimum_nan(A), maximum_nan(A)
return mi_r[1], mi_i[1], mi_r[2], mi_i[2]
end
# --------------------------------------------------------------------------------------------------
"""
doy2date(doy[, year]) -> Date
Compute the date from the Day-Of-Year `doy`. If `year` is ommited we take it to mean the current year.
Both `doy` and `year` can be strings or integers.
"""
function doy2date(doy, year=nothing)
_year = (year === nothing) ? string(Dates.year(now())) : string(year)
n_days = Dates.date2epochdays(Date(_year))
_doy = (isa(doy, Integer)) ? doy : parse(Int64, doy)
n_days += _doy - 1
Dates.epochdays2date(n_days)
end
"""
date2doy(date) -> Integer
Compute the Day-Of-Year (DOY) from `date` that can be a string or a Date/DateTime type. If ommited,
returns today's DOY
"""
date2doy() = dayofyear(now())
date2doy(date::TimeType) = dayofyear(date)
date2doy(date::String) = dayofyear(Date(date))
# --------------------------------------------------------------------------------------------------
"""
yeardecimal(date)
Convert a Date or DateTime or a string representation of them to decimal years, or vice-versa.
That is, convert from decimal year to DateTime.
### Example
yeardecimal(now())
"""
function yeardecimal(dtm::Union{String, Vector{String}})
try
yeardecimal(DateTime.(dtm))
catch
yeardecimal(Date.(dtm))
end
end
function yeardecimal(dtm::Union{Date, Vector{Date}})
year.(dtm) .+ (dayofyear.(dtm) .- 1) ./ daysinyear.(dtm)
end
function yeardecimal(dtm::Union{DateTime, Vector{DateTime}})
Y = year.(dtm)
# FRAC = number_of_milli_sec_in_datetime / number_of_milli_sec_in_that_year
frac = (Dates.datetime2epochms.(dtm) .- Dates.datetime2epochms.(DateTime.(Y))) ./ (daysinyear.(dtm) .* 86400000)
Y .+ frac
end
# This function is from DateFormats.jl
function yeardecimal(years::Real) # Covert from decimal years to DateTime
years_whole = floor(Int, years)
year_ms = DateTime(years_whole + 1) - DateTime(years_whole) |> Dates.value
period_ms = year_ms * (years - years_whole)
return DateTime(years_whole) + Millisecond(round(Int64, period_ms))
end
# --------------------------------------------------------------------------------------------------
"""
settimecol!(D::GMTdataset; col::Int=0, time_system="", time_unit="", time_epoch="")
Set the time column of a `D` GMTdataset (or vector of GMTdatasets) to a given time system.
### Args
- `D`: a GMTdataset or a vector of GMTdatasets
### Kwargs
- `col`: is the column number of the time column. Attention, this is a MANDATORY option
If only `col` is provided (_i.e._, no `time_xxx` options are used), then we only set internal
info to make `col` the time column. That is, we assume that `col` contains already time in seconds (Unix time)
- `time_system`: is one of: "JD", "MJD", "J2000", "S1985", "UNIX", "RD0001", "RATA".
See the GMT manual for a description of these systems (https://docs.generic-mapping-tools.org/dev/gmt.conf.html#term-TIME_SYSTEM).
- `time_epoch`: In ALTERNATIVE to `time_system` use the `time_epoch` AND `time_unit` options.
`time_epoch` a string of the form 'yyyy-mm-ddT[hh:mm:ss]' (Gregorian) or 'yyyy-Www-ddT[hh:mm:ss]' (ISO)
- `time_unit`: is one of: "year", "month", "week", "day", "hour", "minute", "second"
"""
function settimecol!(D::GMTdataset; col::Int=0, time_system="", time_unit="", time_epoch="")
(col <= 0) && error("Must provide the 'col' (column) number of the time column")
(col > size(D, 2)) && error("Column number $col is out of range")
(time_system == "") && ((time_unit != "" && time_epoch == "") || (time_unit == "" && time_epoch != "")) &&
error("Must provide either 'time_system' or, 'time_unit' AND 'time_epoch'")
!(time_system in ["", "JD", "MJD", "J2000", "S1985", "UNIX", "RD0001", "RATA"]) &&
error("time_system must be one of: \"JD\", \"MJD\", \"J2000\", \"S1985\", \"UNIX\", \"RD0001\", \"RATA\"")
(time_epoch != "") && (count_chars(time_epoch, '-') != 2) &&
error("time_epoch must be a string of the form 'yyyy-mm-ddT[hh:mm:ss]' (Gregorian) or 'yyyy-Www-ddT[hh:mm:ss]' (ISO)")
(time_unit != "") && (tu = lowercase(time_unit[1])) ∉ ['y', 'o', 'w', 'd', 'h', 'm', 's'] &&
error("time_unit must be a string starting with 'y' (year), 'o' (month), 'w' (week), 'd' (day), 'h' (hour), 'm' (minute) or 's' (second)")
(startswith(lowercase(time_unit), "mo") && (tu = 'o')) # Because both month and minute start with an 'm'
col_s = string(col)
((Tc = get(D.attrib, "Timecol", "")) == "") ? D.attrib["Timecol"] = col_s : (Tc != col_s) ? D.attrib["Timecol"] = Tc * ",$col" : nothing
D.colnames[col] = "Time"
(time_system == "" && time_unit == "" && time_epoch == "") && return D # Just set the column 'col' as a TimeCol
if (time_system != "") D.attrib["Time_system"] = time_system
else
D.attrib["Time_unit"], D.attrib["Time_epoch"] = string(tu), time_epoch
# The following is to help show_pretty_datasets to print the correct ISO time
(tu == 'y') && startswith(time_epoch, "0000-01-01") && (D.attrib["what_time_sys"] = "YearDecimal0000")
end
return D
end
function settimecol!(D::Vector{GMTdataset}; col::Int=0, time_system="", time_unit="", time_epoch="")
settimecol!(view(D,1), col=col, time_system=time_system, time_unit=time_unit, time_epoch=time_epoch)
end
function settimecol!(x)
@warn "settimecol!() is only implemented for GMTdatasets, not this type of input ($(typeof(x)))"
end
# --------------------------------------------------------------------------------------------------
"""
setdecyear_time!(D::GDtype, col::Int=1)
Convenient function to set column of a GMTdataset that has a decimal year to a `Time` column.
Note that no check is made to ensure the values are indeed decimal years.
"""
function setdecyear_time!(D::GDtype, col::Int=1)
isa(D, GMTdataset) ? settimecol!(D, col=col, time_epoch="0000-01-01", time_unit="year") :
settimecol!(D[1], col=col, time_epoch="0000-01-01", time_unit="year")
end
# --------------------------------------------------------------------------------------------------
function peaks(n::Int=49; N::Int=49, grid::Bool=true, pixreg::Bool=false)
(n != 49 && N == 49) && (N = n)
x,y = meshgrid(range(-3,stop=3,length=N))
z = 3 * (1 .- x).^2 .* exp.(-(x.^2) .- (y .+ 1).^2) .- 10*(x./5 .- x.^3 .- y.^5) .* exp.(-x.^2 .- y.^2)
.- 1/3 * exp.(-(x .+ 1).^2 .- y.^2)
if (grid)
inc = y[2]-y[1]
_x = (pixreg) ? collect(range(-3-inc/2,stop=3+inc/2,length=N+1)) : collect(range(-3,stop=3,length=N))
_y = copy(_x)
z = Float32.(z)
reg = (pixreg) ? 1 : 0
G = GMTgrid("", "", 0, 0, [_x[1], _x[end], _y[1], _y[end], minimum(z), maximum(z)], [inc, inc],
reg, NaN, "", "", "", "", String[], _x, _y, Vector{Float64}(), z, "x", "y", "", "z", "BCB", 1f0, 0f0, 0, 1)
return G
else
return x,y,z
end
end
# -------------------------------------------------------------------------------------------------
# Median absolute deviation. This function is a trimmed version from the StatsBase package
# https://github.com/JuliaStats/StatsBase.jl/blob/60fb5cd400c31d75efd5cdb7e4edd5088d4b1229/src/scalarstats.jl#L527-L536
"""
MAD, median = mad(x)
Compute the median absolute deviation (MAD) of collection `x` around the median
The MAD is multiplied by `1 / quantile(Normal(), 3/4) ≈ 1.4826`, in order to obtain a consistent estimator
of the standard deviation under the assumption that the data is normally distributed. Return also the median
of `x` (used to compute the MAD) as a second output.
"""
mad(x) = mad!(Base.copymutable(x))
function mad!(x::AbstractArray)
mad_constant = 1.4826022185056018
isempty(x) && throw(ArgumentError("mad is not defined for empty arrays"))
have_nans = any(isnan.(x))
c = (have_nans) ? median(skipnan(x)) : median!(x)
T = promote_type(typeof(c), eltype(x))
U = eltype(x)
x2 = U == T ? x : isconcretetype(U) && isconcretetype(T) && sizeof(U) == sizeof(T) ? reinterpret(T, x) : similar(x, T)
x2 .= abs.(x .- c)
return (have_nans) ? median(skipnan(x2)) * mad_constant : median!(x2) * mad_constant, c
end
meshgrid(v::AbstractVector) = meshgrid(v, v)
function meshgrid(vx::AbstractVector{T}, vy::AbstractVector{T}) where T
X = [x for _ in vy, x in vx]
Y = [y for y in vy, _ in vx]
X, Y
end
function meshgrid(vx::AbstractVector{T}, vy::AbstractVector{T}, vz::AbstractVector{T}) where T
m, n, o = length(vy), length(vx), length(vz)
vx = reshape(vx, 1, n, 1)
vy = reshape(vy, m, 1, 1)
vz = reshape(vz, 1, 1, o)
om = ones(Int, m)
on = ones(Int, n)
oo = ones(Int, o)
(vx[om, :, oo], vy[:, on, oo], vz[om, on, :])
end
function ndgrid(v1::AbstractVector{T}, v2::AbstractVector{T}) where {T}
# From https://github.com/ChrisRackauckas/VectorizedRoutines.jl/blob/master/src/matlab.jl
m, n = length(v1), length(v2)
v1 = reshape(v1, m, 1)
v2 = reshape(v2, 1, n)
(repeat(v1, 1, n), repeat(v2, m, 1))
end
"""
accumarray(subs, val, sz=(maximum(subs),))
See MATLAB's documentation for more details.
"""
function accumarray(subs, val, sz=(maximum(subs),))
A = zeros(eltype(val), sz...)
@inbounds for i = 1:numel(val)
A[subs[i]] += val[i]
end
A
end
function accumarray(subs, val::Number, sz=(maximum(subs),))
A = zeros(eltype(val), sz...)
@inbounds for i = 1:numel(subs)
A[subs[i]] += val
end
A
end
# --------------------------------------------------------------------------------------------------
function tic()
t0 = time_ns()
task_local_storage(:TIMERS, (t0, get(task_local_storage(), :TIMERS, ())))
return t0
end
function _toq()
t1 = time_ns()
timers = get(task_local_storage(), :TIMERS, ())
(timers === ()) && error("`toc()` without `tic()`")
t0 = timers[1]::UInt64
task_local_storage(:TIMERS, timers[2])
(t1-t0)/1e9
end
function toc(V=true)
t = _toq()
(V) && println("elapsed time: ", t, " seconds")
return t
end
# --------------------------------------------------------------------------------------------------
"""
isnodata(array::AbstractArray, val=0)
Return a boolean array with the same size a `array` with 1's (`true`) where ``array[i] == val``.
Test with an image have shown that this function was 5x faster than ``ind = (I.image .== 0)``
"""
function isnodata(array::AbstractArray, val=0)
nrows, ncols = size(array,1), size(array,2)
nlayers = (ndims(array) == 3) ? size(array,3) : 1
if (ndims(array) == 3) indNaN = fill(false, nrows, ncols, nlayers)
else indNaN = fill(false, nrows, ncols)
end
@inbounds Threads.@threads for k = 1:nrows * ncols * nlayers # 5x faster than: indNaN = (I.image .== 0)
(array[k] == val) && (indNaN[k] = true) # With Threads.@threads it insists indNaN is a CoreBox but it's 2x faster
end
indNaN
end
#=
function isnodata(array::Matrix{T}, val=0) where T
nrows, ncols = size(array)
indNaN = fill(false, nrows, ncols)
@inbounds Threads.@threads for k = 1:nrows * ncols # 5x faster than: indNaN = (I.image .== 0)
(array[k] == val) && (indNaN[k] = true)
end
indNaN
end
function isnodata(array::Array{T,3}, val=0) where T
nrows, ncols, nlayers = size(array)
indNaN = fill(false, nrows, ncols, nlayers)
@inbounds Threads.@threads for k = 1:nrows * ncols * nlayers # 5x faster than: indNaN = (I.image .== 0)
(array[k] == val) && (indNaN[k] = true)
end
indNaN
end
=#
# ---------------------------------------------------------------------------------------------------
function fakedata(sz...)
# 'Stolen' from Plots.fakedata()
y = zeros(sz...)
for r in 2:size(y,1)
y[r,:] = 0.95 * vec(y[r-1,:]) + randn(size(y,2))
end
y
end
# ---------------------------------------------------------------------------------------------------
"""
delrows(A::Union{Matrix, GDtype}; rows::Vector{<:Integer}=Int[], nodata=nothing, col=0)
Return a new matrix or GMTdataset where the rows listed in the vector of integers `rows`
or rows found with other conditions were deleted.
### Args
- `A`: Matrix or GMTdataset
### Kwargs
- `rows`: Vector of integers specifying the rows to delete. Note: either this or the `nodata` option must be specified.
- `nodata`: Value that indicates no data value. Rows where this value is found are deleted. Use ``nodata=NaN``
to delete rows with NaN values. Note: either this or the `rows` option must be specified.
- `col`: Column to check for `nodata`. the Default is to search in all columns. Use this option is
to restrict the search to a specific column.
### Examples
```julia
A = [1.0:6 10:10:60];
A[3] = NaN;
M1 = delrows(A, rows=[1,5])
M2 = delrows(M1, nodata=NaN)
M3 = delrows(M2, nodata=40, col=2)
```
"""
function delrows(A::Matrix; rows::Vector{<:Integer}=Int[], nodata=nothing, col=0)
isempty(rows) && nodata === nothing && error("Must select something to delete")
@assert col >= 0 && col <= size(A,2) "Column number must be between 1 and $(size(A,2))"
if !isempty(rows)
bv = BitVector(undef, size(A,1))
for k = 1:size(A,1) bv[k] = true end # Because I don't know how to create an initialized BitVector
bv[rows] .= false
else
if (col == 0)
if isnan(nodata)
bv = .!isnan.(view(A,:,1))
for k = 2:size(A,2) bv .= bv .& .!isnan.(view(A,:,k)) end
else
bv = (view(A,:,1) .!= nodata)
for k = 2:size(A,2) bv .= bv .& (view(A,:,k) .!= nodata) end
end
else
if isnan(nodata) bv = isnan.(view(A,:,col))
else bv = (view(A,:,col) .!= nodata)
end
end
end
return any(bv) ? A[bv,:] : A # If nothing found, just return the original array
end
function delrows(D::GMTdataset; rows::Vector{<:Integer}=Int[], nodata=nothing, col=0)
mat = delrows(D.data, rows=rows, nodata=nodata, col=col)
return size(mat,1) != size(D,1) ? mat2ds(mat, D) : D # If nothing found, just return the original dataset
end
function delrows(D::Vector{<:GMTdataset}; rows::Vector{<:Integer}=Int[], nodata=nothing, col=0)
Dv = Vector{GMTdataset{typeof(D[1]),2}}(undef, length(D))
for k = 1:length(D)
Dv[k] = delrows(D[k], rows=rows, nodata=nodata, col=col)
end
return Dv
end
# --------------------------------------------------------------------------------------------------
"""
R = rescale(A; low=0.0, up=1.0; inputmin=NaN, inputmax=NaN, stretch=false, type=nothing)
- `A`: is either a GMTgrid, GMTimage, Matrix{AbstractArray} or a file name. In later case the file is read
with a call to `gmtread` that automatically decides how to read it based on the file extension ... not 100% safe.
- `rescale(A)` rescales all entries of an array `A` to [0,1].
- `rescale(A,low,up)` rescales all entries of A to the interval [low,up].
- `rescale(..., inputmin=imin)` sets the lower bound `imin` for the input range. Input values less
than `imin` will be replaced with `imin`. The default is min(A).
- `rescale(..., inputmax=imax)` sets the lower bound `imax` for the input range. Input values greater
than `imax` will be replaced with `imax`. The default is max(A).
- `rescale(..., stretch=true)` automatically determines [inputmin inputmax] via a call to histogram that
will (try to) find good limits for histogram stretching. The form `stretch=(imin,imax)` allows specifying
the input limits directly.
- `type`: Converts the scaled array to this data type. Valid options are all Unsigned types (e.g. `UInt8`).
Default returns the same data type as `A` if it's an AbstractFloat, or Float64 if `A` is an integer.
Returns a GMTgrid if `A` is a GMTgrid of floats, a GMTimage if `A` is a GMTimage and `type` is used or
an array of Float32|64 otherwise.
"""
function rescale(A::String; low=0.0, up=1.0, inputmin=NaN, inputmax=NaN, stretch=false, type=nothing)
GI = gmtread(A)
rescale(GI; low=low, up=up, inputmin=inputmin, inputmax=inputmax, stretch=stretch, type=type)
end
function rescale(A::AbstractArray; low=0.0, up=1.0, inputmin=NaN, inputmax=NaN, stretch=false, type=nothing)
(type !== nothing && (!isa(type, DataType) || !(type <: Unsigned))) && error("The 'type' variable must be an Unsigned DataType")
((!isnan(inputmin) || !isnan(inputmax)) && stretch == 1) && @warn("The `stretch` option overrules `inputmin|max`.")
low, up = Float64(low), Float64(up)
_inputmin::Float64, _inputmax::Float64 = Float64(inputmin), Float64(inputmax)
if (stretch == 1)
#_inputmin, _inputmax = histogram(A, getauto=true)
if (isa(A, GMTgrid) || (typeof(A) <: SubArray{Float32})) # These functions are defined in pshistogram, but avoid calling it.
hst, inc, = hst_floats(A)
_inputmin, _inputmax = find_histo_limits(A, nothing, inc, hst)
else # Images
_inputmin, _inputmax = find_histo_limits(A, nothing, 20)
end
elseif (isa(stretch, Tuple) || (isvector(stretch) && length(stretch) == 2))
_inputmin, _inputmax = stretch[1], stretch[2]
end
(isnan(_inputmin)) && (mi::Float64 = (isa(A, GItype)) ? A.range[5] : minimum_nan(A))
(isnan(_inputmax)) && (ma::Float64 = (isa(A, GItype)) ? A.range[6] : maximum_nan(A))
_inmin::Float64 = (isnan(_inputmin)) ? mi : _inputmin
_inmax::Float64 = (isnan(_inputmax)) ? ma : _inputmax
d1 = _inmax - _inmin
(d1 <= 0.0) && error("Stretch range has inputmin > inputmax.")
d2 = up - low
sc::Float64 = d2 / d1
have_nans = false
if (eltype(A) <: AbstractFloat) # Float arrays can have NaNs
have_nans = !(isa(A, GMTgrid) && A.hasnans == 1)
have_nans && (have_nans = any(!isfinite, A))
end
if (type !== nothing)
(low != 0.0 || up != 1.0) && (@warn("When converting to Unsigned must have a=0, b=1"); low=0.0; up=1.0)
o = (type == UInt8) ? Array{UInt8}(undef, size(A)) : Array{UInt16}(undef, size(A))
_tmax::Float64 = (type == UInt8) ? typemax(UInt8) : typemax(UInt16)
sc *= _tmax
low *= _tmax
if (have_nans)
if (isnan(_inputmin) && isnan(_inputmax))
if (type == UInt8)
@inbounds for k = 1:numel(A) isnan(A[k]) && (o[k] = 0; continue); o[k] = round(UInt8, low + (A[k] -_inmin) * sc) end
else
@inbounds for k = 1:numel(A) isnan(A[k]) && (o[k] = 0; continue); o[k] = round(UInt16, low + (A[k] -_inmin) * sc) end
end
else
low_i, up_i = round(type, low), round(type, up*typemax(type))
@inbounds for k = 1:numel(A)
isnan(A[k]) && (o[k] = 0; continue)
o[k] = (A[k] < _inmin) ? low_i : ((A[k] > _inmax) ? up_i : round(type, low + (A[k] -_inmin) * sc))
end
end
else
if (isnan(_inputmin) && isnan(_inputmax))
if (type == UInt8)
@inbounds for k = 1:numel(A) o[k] = round(UInt8, low + (A[k] -_inmin) * sc) end
else
@inbounds for k = 1:numel(A) o[k] = round(UInt16, low + (A[k] -_inmin) * sc) end
end
else
low_i, up_i = round(type, low), round(type, up*typemax(type))
@inbounds for k = 1:numel(A)
o[k] = (A[k] < _inmin) ? low_i : ((A[k] > _inmax) ? up_i : round(type, low + (A[k] -_inmin) * sc))
end
end
end
return isa(A, GItype) ? mat2img(o, A) : o
else
oType = (eltype(A) <: AbstractFloat) ? eltype(A) : Float64
o = Array{oType}(undef, size(A))
if (oType <: Integer && have_nans) # Shitty case
if (isnan(_inputmin) && isnan(_inputmax)) # Faster case.
@inbounds for k = 1:numel(A) isnan(A[k]) && (o[k] = 0; continue); o[k] = low + (A[k] -_inmin) * sc end
else
@inbounds for k = 1:numel(A)
isnan(A[k]) && (o[k] = 0; continue)
o[k] = (A[k] < _inmin) ? low : ((A[k] > _inmax) ? up : low + (A[k] -_inmin) * sc)
end
end
else
if (isnan(_inputmin) && isnan(_inputmax)) # Faster case. No IFs in loop
@inbounds for k = 1:numel(A) o[k] = low + (A[k] -_inmin) * sc end
else
@inbounds for k = 1:numel(A)
o[k] = (A[k] < _inmin) ? low : ((A[k] > _inmax) ? up : low + (A[k] -_inmin) * sc)
end
end end
return isa(A, GItype) ? mat2grid(o, A) : o
end
end
# --------------------------------------------------------------------------------------------------
"""
M = magic(n::Int) => Matrix{Int}
M = magic(n) returns an n-by-n matrix constructed from the integers 1 through n^2 with equal row and column sums.
The order n must be a scalar greater than or equal to 3 in order to create a valid magic square.
"""
function magic(n::Int)
# From: https://gist.github.com/phillipberndt/2db94bf5e0c16161dedc
# Had to suffer with Julia painful matrix indexing system to make it work. Gives the same as magic.m
if n % 2 == 1
p = (1:n)
M = n * mod.(p .+ (p' .- div(n+3, 2)), n) .+ mod.(p .+ (2p' .- 2), n) .+ 1
elseif n % 4 == 0
J = div.((1:n) .% 4, 2)
K = J' .== J
M = collect(1:n:(n*n)) .+ reshape(0:n-1, 1, n) # Is it really true that we can't make a 1 row matix?????
M[K] .= n^2 .+ 1 .- M[K]
else
p = div(n, 2)
M = magic(p)
M = [M M .+ 2p^2; M .+ 3p^2 M .+ p^2]
(n == 2) && return M
i = (1:p)
k = Int((n-2)/4)
j = convert(Array{Int}, [(1:k); ((n-k+2):n)])
M[[i; i.+p],j] = M[[i.+p; i],j]
ii = k+1
j = [1; ii]
M[[ii; ii+p],j] = M[[ii+p; ii],j]
end
return M
end
"""
setfld!(D, kwargs...)
Sets fields of GMTdataset (or a vector of it), GMTgrid and GMTimage. Field names and field
values are transmitted via `kwargs`
Example:
setfld!(D, geom=wkbPolygon)
"""
function setfld!(D::Union{GMTgrid, GMTimage, GMTdataset}; kwargs...)
for key in keys(kwargs)
setfield!(D, key, kwargs[key])
end
end
function setfld!(D::Vector{<:GMTdataset}; kwargs...)
for d in D
setfld!(d; kwargs...)
end
end
# ---------------------------------------------------------------------------------------------------
function get_group_indices(d::Dict, data)::Tuple{Vector{Vector{Union{Int,String}}}, Vector{Any}}
# If 'data' is a Vector{GMTdataset} return results respecting only the first GMTdataset of the array.
group::Vector{Union{Int,String}} = ((val = find_in_dict(d, [:group])[1]) === nothing) ? Int[] : val
groupvar::StrSymb = ((val = find_in_dict(d, [:groupvar :hue])[1]) === nothing) ? "" : val
((groupvar != "") && !isa(data, GDtype)) && error("'groupvar' can only be used when input is a GMTdataset")
(isempty(group) && groupvar == "") && return Int[], []
get_group_indices(isa(data, Vector{<:GMTdataset}) ? data[1] : data, group, groupvar)
end
function get_group_indices(data, group, groupvar::StrSymb)::Tuple{Vector{Vector{Union{Int,String}}}, Vector{Any}}
# This function is not meant for public consuption. In fact it's only called directly by the parallelplot() fun
# or indirectly, via the other method, by plot().
# Returns a Vector of AbstractVectors with the indices of each group and a Vector of names or numbers
(isempty(group) && isa(data, GMTdataset) && groupvar == "text") && (group = data.text)
(!isempty(group) && length(group) != size(data,1)) && error("Length of `group` and number of rows in input don't match.")
if (isempty(group) && isa(data, GMTdataset))
isa(groupvar, Integer) && (group = Base.invokelatest(view, data.data, :, groupvar))
if (isempty(group))
gvar = string(groupvar)
x = (gvar .== data.colnames) # Try also to fish the name of the text column
any(x) && (group = x[end] ? data.text : Base.invokelatest(view, data, :, findfirst(x)))
end
end
gidx, gnames = !isempty(group) ? grp2idx(group) : ([1:size(data,1)], [0])
end
# ----------------------------------------------------------------------------------------------------------
"""
gidx, gnames = grp2idx(S::AbstracVector)
Creates an index Vector{Vector} from the grouping variable S. S can be an AbstracVector of elements
for which the `==` method is defined. It returns a Vector of Vectors with the indices of the elements
of each group. There will be as many groups as `length(gidx)`. `gnames` is a string vector holding
the group names.
"""
function grp2idx(s)
gnames = unique(s)
gidx = [findall(s .== gnames[k]) for k = 1:numel(gnames)]
gidx, gnames
end
# ----------------------------------------------------------------------------------------------------------
"""
"""
function replicateline(xy, d)
# https://stackoverflow.com/questions/1250419/finding-points-on-a-line-with-a-given-distance
# magnitude = (1^2 + m^2)^(1/2)
# N = <1, m> / magnitude = <1 / magnitude, m / magnitude>
# f(t) = A + t*N
line2 = Matrix{Float64}(undef, size(xy))
m1 = -(xy[2,1] - xy[1,1]) / (xy[2,2] - xy[1,2]) # Slope of the perpendicular to first segment
inv_mag_d = d / sqrt(1 + m1*m1)
line2[1, 1] = xy[1, 1] + inv_mag_d
line2[1, 2] = xy[1, 2] + m1 * inv_mag_d
@inbounds for k = 2:size(xy,1)-1
m2 = -(xy[k,1] - xy[k-1,1]) / (xy[k,2] - xy[k-1,2]) # Slope of the perpendicular to line segment k -> k+1
m = (m1 + m2) / 2 # ...
inv_mag_d = d / sqrt(1 + m*m)
line2[k, 1] = xy[k, 1] + inv_mag_d
line2[k, 2] = xy[k, 2] + m * inv_mag_d
m1 = m2
end
m1 = -(xy[end,1] - xy[end-1,1]) / (xy[end,2] - xy[end-1,2]) # Slope of the perpendicular to last segment
inv_mag_d = d / sqrt(1 + m1*m1)
line2[end, 1] = xy[end, 1] + inv_mag_d
line2[end, 2] = xy[end, 2] + m1 * inv_mag_d
return line2
end
# ---------------------------------------------------------------------------------------------------
"""
height (nrows), width (ncols) = dims(GI::GItype)
Return the width and height of the grid/cube or image. The difference from `size` is that
the when the memory layout is 'rows' the array is transposed and we get the wrong info. Here,
we use the sizes of the 'x,y' coordinate vectors to determine the array's true shape.
"""
dims(GI::GItype) = (GI.layout != "" && GI.layout[2] == 'C') ? (size(GI,1), size(GI,2)) : (length(GI.y), length(GI.x)) .- GI.registration
# EDIPO SECTION
# ---------------------------------------------------------------------------------------------------
linspace(start, stop, length=100) = range(start, stop=stop, length=length)
logspace(start, stop, length=100) = exp10.(range(start, stop=stop, length=length))
fields(arg) = fieldnames(typeof(arg))
fields(arg::Array) = fieldnames(typeof(arg[1]))
flipud(A) = reverse(A, dims=1)
fliplr(A) = reverse(A, dims=2)
flipdim(A,dim) = reverse(A, dims=dim)
flipdim!(A,dim) = reverse!(A, dims=dim)
#feval(fn_str, args...) = eval(Symbol(fn_str))(args...)
numel(args...)::Int = length(args...)
dec2bin(n::Integer, mindigits::Int=0) = string(n, base=2, pad=mindigits)
bin2dec(b::Union{AbstractString, Char}) = parse(Int, b, base=2)
function sub2ind(sz, args...)
linidx = LinearIndices(sz)
getindex.([linidx], args...)
end
function fileparts(fn::String)
pato, ext = splitext(fn)
pato, fname = splitdir(pato)
return pato, fname, ext
end
# ---------------------------------------------------------------------------------------------------
"""
tests(fname::String, subdir::String="assets") -> String
Return the full path of a test file `fname` under `TESTSDIR/subdir`.
If `fname` has no extension, search for a matching file in the subdirectory.
If no match is found, issue a warning and return `""`.