diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index 822ee4676..dcab1c74b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -2265,7 +2265,15 @@ public partial interface IDatabase : IRedis, IDatabaseAsync /// Whether to exclude and from the range check (defaults to both inclusive). /// The flags to use for this operation. /// The number of elements in the specified score range. - /// + /// + /// + /// The bounds are compared by their raw bytes, the way Redis orders a lexicographical range, and are swapped if + /// sorts after , so the count matches what + /// returns. + /// A bound that looks like a number, such as "0123" or 10, is ordered as text, so "10" sorts before "2". + /// + /// + /// long SortedSetLengthByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None); /// @@ -2461,7 +2469,14 @@ SortedSetEntry[] SortedSetRangeByScoreWithScores( /// How many items to take. /// The flags to use for this operation. /// List of elements in the specified score range. - /// + /// + /// + /// The bounds are compared by their raw bytes, the way Redis orders a lexicographical range, and are swapped if + /// sorts after (an excluded bound stays excluded). + /// A bound that looks like a number, such as "0123" or 10, is ordered as text, so "10" sorts before "2". + /// + /// + /// RedisValue[] SortedSetRangeByValue( RedisKey key, RedisValue min, @@ -2485,9 +2500,16 @@ RedisValue[] SortedSetRangeByValue( /// The flags to use for this operation. /// List of elements in the specified score range. /// + /// + /// The bounds are compared by their raw bytes, the way Redis orders a lexicographical range, and are swapped if + /// sorts after (an excluded bound stays excluded). + /// A bound that looks like a number, such as "0123" or 10, is ordered as text, so "10" sorts before "2". + /// + /// /// See /// , /// . + /// /// RedisValue[] SortedSetRangeByValue( RedisKey key, @@ -2571,7 +2593,15 @@ RedisValue[] SortedSetRangeByValue( /// Which of and to exclude (defaults to both inclusive). /// The flags to use for this operation. /// The number of elements removed. - /// + /// + /// + /// Unlike , + /// the bounds are not reordered: if sorts after , nothing is removed. + /// Redis compares them by their raw bytes, so a bound that looks like a number is ordered as text: + /// "2" to "10" removes nothing, because "10" sorts before "2". + /// + /// + /// long SortedSetRemoveRangeByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None); /// diff --git a/src/StackExchange.Redis/RedisDatabase.cs b/src/StackExchange.Redis/RedisDatabase.cs index 3c9c28918..2c0cce6ac 100644 --- a/src/StackExchange.Redis/RedisDatabase.cs +++ b/src/StackExchange.Redis/RedisDatabase.cs @@ -5848,6 +5848,7 @@ private Message GetLexMessage(RedisCommand command, RedisKey key, RedisValue min public long SortedSetLengthByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) { + ReverseLimits(Order.Ascending, ref exclude, ref min, ref max); // count what SortedSetRangeByValue would return var msg = GetLexMessage(RedisCommand.ZLEXCOUNT, key, min, max, exclude, 0, -1, flags, Order.Ascending); return ExecuteSync(msg, ResultProcessor.Int64); } @@ -5857,7 +5858,8 @@ public RedisValue[] SortedSetRangeByValue(RedisKey key, RedisValue min, RedisVal private static void ReverseLimits(Order order, ref Exclude exclude, ref RedisValue start, ref RedisValue stop) { - bool reverseLimits = (order == Order.Ascending) == (stop != default && start.CompareTo(stop) > 0); + // order by raw bytes, as redis does; CompareTo would read numeric-looking text such as "0123" as a number + bool reverseLimits = (order == Order.Ascending) == (stop != default && RedisValue.LexCompare(start, stop) > 0); if (reverseLimits) { var tmp = start; @@ -5887,12 +5889,15 @@ public RedisValue[] SortedSetRangeByValue( public long SortedSetRemoveRangeByValue(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) { + // deliberately not reordered (unlike range/length): bounds the wrong way round remove nothing, rather than + // everything between them as text - "2" and "10" would otherwise also remove "15" and "100" var msg = GetLexMessage(RedisCommand.ZREMRANGEBYLEX, key, min, max, exclude, 0, -1, flags, Order.Ascending); return ExecuteSync(msg, ResultProcessor.Int64); } public Task SortedSetLengthByValueAsync(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) { + ReverseLimits(Order.Ascending, ref exclude, ref min, ref max); // count what SortedSetRangeByValueAsync would return var msg = GetLexMessage(RedisCommand.ZLEXCOUNT, key, min, max, exclude, 0, -1, flags, Order.Ascending); return ExecuteAsync(msg, ResultProcessor.Int64); } @@ -5917,6 +5922,7 @@ public Task SortedSetRangeByValueAsync( public Task SortedSetRemoveRangeByValueAsync(RedisKey key, RedisValue min, RedisValue max, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) { + // deliberately not reordered; see SortedSetRemoveRangeByValue var msg = GetLexMessage(RedisCommand.ZREMRANGEBYLEX, key, min, max, exclude, 0, -1, flags, Order.Ascending); return ExecuteAsync(msg, ResultProcessor.Int64); } diff --git a/src/StackExchange.Redis/RedisValue.cs b/src/StackExchange.Redis/RedisValue.cs index 0546216ee..ec0073e6d 100644 --- a/src/StackExchange.Redis/RedisValue.cs +++ b/src/StackExchange.Redis/RedisValue.cs @@ -407,6 +407,34 @@ private static int BlobCompareTo(in RedisValue x, in RedisValue y) return x.UnsafeRawSpan(out xScratch).SequenceCompareTo(y.UnsafeRawSpan(out yScratch)); } + // byte-wise ordinal comparison of any two values by their raw bytes, which is how redis orders a lex range; + // unlike CompareTo, numbers compare as their text ("10" and 10 both sort before "2", and "0123" is not 123), + // and null and empty are both no bytes, so they sort first + internal static int LexCompare(in RedisValue x, in RedisValue y) + { + try + { + if (IsBlob(x.Type) && IsBlob(y.Type)) return BlobCompareTo(x, y); // both byte-backed: no copies + + byte[]? xLease = null, yLease = null; + try + { + return x.AsMemory(out xLease).Span.SequenceCompareTo(y.AsMemory(out yLease).Span); + } + finally + { + if (xLease is not null) ArrayPool.Shared.Return(xLease); + if (yLease is not null) ArrayPool.Shared.Return(yLease); + } + } + catch (Exception ex) + { + // never throws: like CompareTo, a value that can't be read as bytes is treated as equal + ConnectionMultiplexer.TraceWithoutContext(ex.Message); + return 0; + } + } + // true if 'whole' starts with the bytes of 'prefix', for any combination of byte-backed kinds private static bool BlobStartsWith(in RedisValue whole, in RedisValue prefix) { diff --git a/tests/StackExchange.Redis.Tests/LexTests.cs b/tests/StackExchange.Redis.Tests/LexTests.cs index b70fdda7e..afe1512fb 100644 --- a/tests/StackExchange.Redis.Tests/LexTests.cs +++ b/tests/StackExchange.Redis.Tests/LexTests.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System.Text; +using System.Threading.Tasks; using Xunit; namespace StackExchange.Redis.Tests; @@ -56,6 +57,253 @@ public async Task QueryRangeAndLengthByLex() Equate(set, set.Length, "g", "f", "e"); } + [Fact] + public async Task QueryRangeByLexWithNumericPrefix() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("0122", 0), + new SortedSetEntry("0123", 0), + new SortedSetEntry("0123abc", 0), + new SortedSetEntry("0123xyz", 0), + new SortedSetEntry("0124", 0), + new SortedSetEntry("123abc", 0), + ], + CommandFlags.FireAndForget); + + // prefix + '\xff' is the usual autocomplete upper bound; "0123" on its own looks like the number 123 (#1302) + var set = db.SortedSetRangeByValue(key, "0123", "0123\xff"); + var count = db.SortedSetLengthByValue(key, "0123", "0123\xff"); + Equate(set, count, "0123", "0123abc", "0123xyz"); + + set = await db.SortedSetRangeByValueAsync(key, "0123", "0123\xff"); + Equate(set, set.Length, "0123", "0123abc", "0123xyz"); + + set = db.SortedSetRangeByValue(key, "0123", "0123\xff", Exclude.None, Order.Descending); + Equate(set, set.Length, "0123xyz", "0123abc", "0123"); + + set = await db.SortedSetRangeByValueAsync(key, "0123", "0123\xff", Exclude.None, Order.Descending); + Equate(set, set.Length, "0123xyz", "0123abc", "0123"); + } + + [Fact] + public async Task QueryRangeByLexWithNumericPrefixAndExclude() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("0122", 0), + new SortedSetEntry("0123", 0), + new SortedSetEntry("0123abc", 0), + new SortedSetEntry("0123xyz", 0), + new SortedSetEntry("0124", 0), + ], + CommandFlags.FireAndForget); + + var set = db.SortedSetRangeByValue(key, "0123", "0123xyz", Exclude.Start); + var count = db.SortedSetLengthByValue(key, "0123", "0123xyz", Exclude.Start); + Equate(set, count, "0123abc", "0123xyz"); + + set = db.SortedSetRangeByValue(key, "0123", "0123xyz", Exclude.Stop); + count = db.SortedSetLengthByValue(key, "0123", "0123xyz", Exclude.Stop); + Equate(set, count, "0123", "0123abc"); + + set = db.SortedSetRangeByValue(key, "0123", "0123xyz", Exclude.Start, Order.Descending); + Equate(set, set.Length, "0123xyz", "0123abc"); + + set = db.SortedSetRangeByValue(key, "0123", "0123xyz", Exclude.Stop, Order.Descending); + Equate(set, set.Length, "0123abc", "0123"); + } + + [Fact] + public async Task QueryRangeByLexWithDigitOnlyBounds() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("1", 0), + new SortedSetEntry("10", 0), + new SortedSetEntry("15", 0), + new SortedSetEntry("2", 0), + new SortedSetEntry("20", 0), + new SortedSetEntry("3", 0), + ], + CommandFlags.FireAndForget); + + // as text "10" sorts before "2", so the bounds are ordered the way redis orders them + var set = db.SortedSetRangeByValue(key, "2", "10"); + var raw = (RedisValue[]?)db.Execute("ZRANGEBYLEX", [key, "[10", "[2"]); + Assert.Equal(raw, set); + Equate(set, set.Length, "10", "15", "2"); + + set = db.SortedSetRangeByValue(key, "2", "10", Exclude.None, Order.Descending); + raw = (RedisValue[]?)db.Execute("ZREVRANGEBYLEX", [key, "[2", "[10"]); + Assert.Equal(raw, set); + Equate(set, set.Length, "2", "15", "10"); + } + + [Fact] + public async Task QueryRangeByLexWithIntegerBounds() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("1", 0), + new SortedSetEntry("10", 0), + new SortedSetEntry("15", 0), + new SortedSetEntry("2", 0), + new SortedSetEntry("20", 0), + new SortedSetEntry("3", 0), + ], + CommandFlags.FireAndForget); + + // integers are sent as their text, so they are ordered as text too: 10 before 2 + RedisValue min = 2; + RedisValue max = 10; + var set = db.SortedSetRangeByValue(key, min, max); + var raw = (RedisValue[]?)db.Execute("ZRANGEBYLEX", [key, "[10", "[2"]); + Assert.Equal(raw, set); + Equate(set, set.Length, "10", "15", "2"); + + set = db.SortedSetRangeByValue(key, min, max, Exclude.None, Order.Descending); + raw = (RedisValue[]?)db.Execute("ZREVRANGEBYLEX", [key, "[2", "[10"]); + Assert.Equal(raw, set); + Equate(set, set.Length, "2", "15", "10"); + } + + [Fact] + public async Task QueryRangeByLexWithNumericPrefixAsBytes() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("0122", 0), + new SortedSetEntry("0123", 0), + new SortedSetEntry("0123abc", 0), + new SortedSetEntry("0123xyz", 0), + new SortedSetEntry("0124", 0), + new SortedSetEntry("123abc", 0), + ], + CommandFlags.FireAndForget); + + // the same bytes the string bounds "0123" and "0123\xff" are sent as + RedisValue min = Encoding.UTF8.GetBytes("0123"); + RedisValue max = Encoding.UTF8.GetBytes("0123\xff"); + var set = db.SortedSetRangeByValue(key, min, max); + var count = db.SortedSetLengthByValue(key, min, max); + Equate(set, count, "0123", "0123abc", "0123xyz"); + + set = db.SortedSetRangeByValue(key, min, max, Exclude.None, Order.Descending); + Equate(set, set.Length, "0123xyz", "0123abc", "0123"); + + // a string bound against a byte[] bound + set = db.SortedSetRangeByValue(key, "0123", max); + Equate(set, set.Length, "0123", "0123abc", "0123xyz"); + + set = db.SortedSetRangeByValue(key, "0123", max, Exclude.None, Order.Descending); + Equate(set, set.Length, "0123xyz", "0123abc", "0123"); + } + + [Fact] + public async Task LengthByLexMatchesRangeWithDigitOnlyBounds() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("1", 0), + new SortedSetEntry("10", 0), + new SortedSetEntry("15", 0), + new SortedSetEntry("2", 0), + new SortedSetEntry("20", 0), + new SortedSetEntry("3", 0), + ], + CommandFlags.FireAndForget); + + // the count is of the same range SortedSetRangeByValue returns: "10" up to "2", as text + var set = db.SortedSetRangeByValue(key, "2", "10"); + var count = db.SortedSetLengthByValue(key, "2", "10"); + Equate(set, count, "10", "15", "2"); + + count = await db.SortedSetLengthByValueAsync(key, "2", "10"); + Equate(set, count, "10", "15", "2"); + + // swapping the bounds swaps which end is excluded, as it does for the range + set = db.SortedSetRangeByValue(key, "2", "10", Exclude.Start); + count = db.SortedSetLengthByValue(key, "2", "10", Exclude.Start); + Equate(set, count, "10", "15"); + } + + [Fact] + public async Task RemoveRangeByLexDoesNotReorderBounds() + { + await using var conn = Create(); + + var db = conn.GetDatabase(); + RedisKey key = Me(); + db.KeyDelete(key, CommandFlags.FireAndForget); + + db.SortedSetAdd( + key, + [ + new SortedSetEntry("1", 0), + new SortedSetEntry("10", 0), + new SortedSetEntry("15", 0), + new SortedSetEntry("2", 0), + new SortedSetEntry("20", 0), + new SortedSetEntry("3", 0), + ], + CommandFlags.FireAndForget); + + // deliberate: unlike range and length, removal is not reordered, so bounds the wrong way round remove + // nothing - reordering "2" and "10" would remove "10" and "15" as well, which a caller thinking of + // numbers never asked for + long removed = db.SortedSetRemoveRangeByValue(key, "2", "10"); + Assert.Equal(0, removed); + + removed = await db.SortedSetRemoveRangeByValueAsync(key, "2", "10"); + Assert.Equal(0, removed); + + var set = db.SortedSetRangeByRank(key); + Equate(set, set.Length, "1", "10", "15", "2", "20", "3"); + } + [Fact] public async Task RemoveRangeByLex() { diff --git a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs index 6f8208ea5..468c8cc5e 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs @@ -970,4 +970,103 @@ public void BinaryComparer_WorksAsADictionaryComparer() Assert.Equal("two", dictionary["beta"]); Assert.False(dictionary.ContainsKey("gamma")); } + + [Fact] + // LexCompare orders values by the bytes they are sent as, which is how redis orders a lex range (#1302). + // CompareTo reads numeric-looking text as a number, so the two are expected to disagree on digits. + public void LexCompare_OrdersByRawBytes() + { + // digits are text: "10" sorts before "2", and "0123" is not the number 123 + Assert.True(RedisValue.LexCompare("10", "2") < 0, LineNumber()); + Assert.True(RedisValue.LexCompare("2", "10") > 0, LineNumber()); + Assert.True(RedisValue.LexCompare("0123", "0123\xff") < 0, LineNumber()); + Assert.True(RedisValue.LexCompare("0123\xff", "0123") > 0, LineNumber()); + Assert.True(((RedisValue)"0123").CompareTo("0123\xff") > 0, LineNumber()); // CompareTo: 123 vs text + + // integers compare as their text too, where CompareTo compares them as numbers + Assert.True(RedisValue.LexCompare(10L, 2L) < 0, LineNumber()); + Assert.True(RedisValue.LexCompare(2L, 10L) > 0, LineNumber()); + Assert.True(((RedisValue)10L).CompareTo(2L) > 0, LineNumber()); + + // a shorter prefix sorts first, and equal bytes are equal + Assert.True(RedisValue.LexCompare("ab", "abc") < 0, LineNumber()); + Assert.Equal(0, RedisValue.LexCompare("abc", "abc")); + + // bytes, not UTF-16: U+FFFD is EF BF BD and U+1F600 is F0 9F 98 80, the opposite of their char order + Assert.True(RedisValue.LexCompare("\uFFFD", "\U0001F600") < 0, LineNumber()); + Assert.True(string.CompareOrdinal("\uFFFD", "\U0001F600") > 0, LineNumber()); + + // a byte[] against a string answers as the two strings would + int expected = Math.Sign(RedisValue.LexCompare("0123", "0123\xff")); + Assert.Equal(expected, Math.Sign(RedisValue.LexCompare(Encoding.UTF8.GetBytes("0123"), "0123\xff"))); + Assert.Equal(expected, Math.Sign(RedisValue.LexCompare("0123", Encoding.UTF8.GetBytes("0123\xff")))); + Assert.Equal(0, RedisValue.LexCompare(Encoding.UTF8.GetBytes("\U0001F600"), "\U0001F600")); + + // null and empty are both no bytes: equal to each other, and before anything else + RedisValue[] nothing = [RedisValue.Null, RedisValue.EmptyString, Array.Empty()]; + RedisValue[] something = ["a", 0, Bytes("a"u8), RedisValue.FromRaw("a"u8), "\0"]; + foreach (var x in nothing) + { + foreach (var y in nothing) + { + Assert.Equal(0, RedisValue.LexCompare(x, y)); + } + foreach (var y in something) + { + Assert.True(RedisValue.LexCompare(x, y) < 0, $"{x.Type} vs {y.Type}"); + Assert.True(RedisValue.LexCompare(y, x) > 0, $"{y.Type} vs {x.Type}"); + } + } + } + + [Fact] + // The answer cannot depend on how a value is stored: two byte-backed values are compared in place, anything + // else is turned into bytes first, and the two routes must agree. Each pair below is the other way round as + // numbers, so a numeric reading anywhere would show. + public void LexCompare_DoesNotDependOnHowAValueIsStored() + { + var kinds = new HashSet(); + + Check( + [100, "100", RedisValue.FromRaw("100"u8), Bytes("100"u8), Bytes("10"u8, "0"u8), Managed("100"u8)], + [20, "20", RedisValue.FromRaw("20"u8), Bytes("20"u8), Bytes("2"u8, "0"u8), Managed("20"u8)]); + + // the numeric kinds those can't reach: a double that isn't whole, and a ulong above long.MaxValue + Check([100.5, "100.5", Bytes("100.5"u8)], [20.5, "20.5", Bytes("20.5"u8)]); + Check([ulong.MaxValue, "18446744073709551615", Bytes("18446744073709551615"u8)], [2, "2", Bytes("2"u8)]); + + RedisValue.StorageType[] expected = + [ + RedisValue.StorageType.Int64, + RedisValue.StorageType.UInt64, + RedisValue.StorageType.Double, + RedisValue.StorageType.String, + RedisValue.StorageType.ShortBlob, + RedisValue.StorageType.ByteArray, + RedisValue.StorageType.Sequence, + RedisValue.StorageType.MemoryManager, + ]; + foreach (var kind in expected) + { + Assert.Contains(kind, kinds); + } + + void Check(RedisValue[] lower, RedisValue[] higher) + { + foreach (var x in lower) + { + kinds.Add(x.Type); + foreach (var y in higher) + { + kinds.Add(y.Type); + Assert.True(RedisValue.LexCompare(x, y) < 0, $"{x} ({x.Type}) vs {y} ({y.Type})"); + Assert.True(RedisValue.LexCompare(y, x) > 0, $"{y} ({y.Type}) vs {x} ({x.Type})"); + } + foreach (var y in lower) + { + Assert.True(RedisValue.LexCompare(x, y) == 0, $"{x} ({x.Type}) vs {y} ({y.Type})"); + } + } + } + } }