From 075edf59adbaa2e7c7e358ae1f0fc5f8e34b2e93 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 09:38:28 +0000 Subject: [PATCH] Fix timezone handling of the date filters in the list query Time based filters showed entries of the neighbouring day whenever the device timezone was not UTC: the day boundaries were calculated with DateTimeUtils.getTodayAsLong(), which returns midnight UTC. That is the correct representation for all-day entries (they are stored as midnight UTC), but entries with a time are stored as the epoch millis of the actual instant, so e.g. "Started today" covered "today 00:00 UTC" until "tomorrow 00:00 UTC" - in a +11:30 timezone that is local 11:30 today until 11:30 tomorrow. The day boundaries are now calculated for both cases separately and the query picks the matching one via the timezone column of the respective date, so all-day entries keep being compared in UTC while all other entries are compared against the beginning of the local day. This applies to the today/tomorrow/within 7 days filters, the in past/future filters (all-day entries of today are now neither), the date range and day range filters (the pickers deliver midnight UTC as well) and to the lookup of the next recurring instance. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Am92mRS8fz3WKPdBSNdVKS --- .../techbee/jtx/database/views/ICal4List.kt | 92 +++++++++++++++---- .../java/at/techbee/jtx/util/DateTimeUtils.kt | 31 ++++++- .../jtx/database/views/ICal4ListTest.kt | 91 ++++++++++++++++++ .../at/techbee/jtx/util/DateTimeUtilsTest.kt | 21 +++++ 4 files changed, 215 insertions(+), 20 deletions(-) create mode 100644 app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt diff --git a/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt b/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt index cd8759456..a3336b4f2 100644 --- a/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt +++ b/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt @@ -65,6 +65,7 @@ import at.techbee.jtx.database.COLUMN_URL import at.techbee.jtx.database.Classification import at.techbee.jtx.database.Component import at.techbee.jtx.database.ICalCollection.Factory.LOCAL_ACCOUNT_TYPE +import at.techbee.jtx.database.ICalObject.Companion.TZ_ALLDAY import at.techbee.jtx.database.Module import at.techbee.jtx.database.Status import at.techbee.jtx.database.TABLE_NAME_COLLECTION @@ -92,8 +93,8 @@ import at.techbee.jtx.database.properties.TABLE_NAME_RELATEDTO import at.techbee.jtx.database.properties.TABLE_NAME_RESOURCE import at.techbee.jtx.ui.list.AnyAllNone import at.techbee.jtx.util.DateTimeUtils +import java.time.LocalDate import java.util.UUID -import kotlin.time.Duration.Companion.days const val VIEW_NAME_ICAL4LIST = "ical4list" @@ -309,6 +310,57 @@ data class ICal4List( isReadOnly = true ) + /** + * jtx stores all-day entries as midnight UTC, while entries with a time are stored as the + * epoch milliseconds of the actual instant. Filtering both with the same UTC based + * boundaries makes entries of the neighbouring day slip into the filter on every device + * that is not set to UTC, so the boundaries have to be calculated for each of the two + * cases separately. + * + * @param dateColumn the column with the date(-time) to filter, e.g. [COLUMN_DTSTART] + * @param timezoneColumn the column with the timezone of [dateColumn], e.g. [COLUMN_DTSTART_TIMEZONE] + * @param from the first day of the filtered range (inclusive) or null for no lower boundary + * @param until the last day of the filtered range (inclusive) or null for no upper boundary + * @return the condition to add to the query + */ + private fun getDayRangeFilter( + dateColumn: String, + timezoneColumn: String, + from: LocalDate?, + until: LocalDate? + ): String { + fun boundaries(startOfDay: (LocalDate) -> Long): String { + val conditions = mutableListOf() + from?.let { conditions.add("$dateColumn >= ${startOfDay(it)}") } + until?.let { conditions.add("$dateColumn < ${startOfDay(it.plusDays(1))}") } + return if (conditions.isEmpty()) "$dateColumn IS NOT NULL" else conditions.joinToString(separator = " AND ") + } + + return "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN ${boundaries(DateTimeUtils::getStartOfDayUTCAsLong)} " + + "ELSE ${boundaries(DateTimeUtils::getStartOfDayLocalAsLong)} END)" + } + + /** + * All-day entries have no time, they are therefore compared against the beginning of today + * (in UTC, see [getDayRangeFilter]) instead of the current time. An all-day entry of today + * is consequently neither in the past nor in the future. + * + * @return the condition that the given date(-time) lies before now + */ + private fun getBeforeNowFilter(dateColumn: String, timezoneColumn: String) = + "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN $dateColumn < ${DateTimeUtils.getStartOfDayUTCAsLong()} " + + "ELSE $dateColumn < ${System.currentTimeMillis()} END)" + + /** + * @return the condition that the given date(-time) lies after now, see [getBeforeNowFilter] + */ + private fun getAfterNowFilter(dateColumn: String, timezoneColumn: String) = + "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN $dateColumn >= ${DateTimeUtils.getStartOfDayUTCAsLong(LocalDate.now().plusDays(1))} " + + "ELSE $dateColumn > ${System.currentTimeMillis()} END)" + fun constructQuery( modules: List, searchCategories: List = emptyList(), @@ -470,27 +522,29 @@ data class ICal4List( if (isExcludeDone) queryString += "AND $COLUMN_PERCENT IS NOT 100 AND ($COLUMN_STATUS IS NULL OR $COLUMN_STATUS NOT IN ('${Status.COMPLETED.status}', '${Status.CANCELLED.status}')) " + val today = LocalDate.now() + val dateQuery = mutableListOf() if (isFilterStartInPast) - dateQuery.add("$COLUMN_DTSTART < ${System.currentTimeMillis()}") + dateQuery.add(getBeforeNowFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE)) if (isFilterStartToday) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today, today)) if (isFilterStartTomorrow) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (2).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today.plusDays(1), today.plusDays(1))) if (isFilterStartWithin7Days) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (8).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today, today.plusDays(7))) if (isFilterStartFuture) - dateQuery.add("$COLUMN_DTSTART > ${System.currentTimeMillis()}") + dateQuery.add(getAfterNowFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE)) if (isFilterOverdue) - dateQuery.add("$COLUMN_DUE < ${System.currentTimeMillis()}") + dateQuery.add(getBeforeNowFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE)) if (isFilterDueToday) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today, today)) if (isFilterDueTomorrow) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (2).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today.plusDays(1), today.plusDays(1))) if (isFilterDueWithin7Days) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (8).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today, today.plusDays(7))) if (isFilterDueFuture) - dateQuery.add("$COLUMN_DUE > ${System.currentTimeMillis()}") + dateQuery.add(getAfterNowFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE)) if (isFilterNoDatesSet) dateQuery.add("$COLUMN_DTSTART IS NULL AND $COLUMN_DUE IS NULL AND $COLUMN_COMPLETED IS NULL ") if (isFilterNoStartDateSet) @@ -503,21 +557,21 @@ data class ICal4List( if (dateQuery.isNotEmpty()) queryString += " AND (${dateQuery.joinToString(separator = " OR ")}) " - // DATE RANGE + // DATE RANGE (the date pickers deliver the selected days as midnight UTC) if(filterStartRangeStart != null || filterStartRangeEnd != null) - queryString += " AND ($COLUMN_DTSTART BETWEEN ${filterStartRangeStart?:Long.MIN_VALUE} AND ${filterStartRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, filterStartRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterStartRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " if(filterDueRangeStart != null || filterDueRangeEnd != null) - queryString += " AND ($COLUMN_DUE BETWEEN ${filterDueRangeStart?:Long.MIN_VALUE} AND ${filterDueRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, filterDueRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterDueRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " if(filterCompletedRangeStart != null || filterCompletedRangeEnd != null) - queryString += " AND ($COLUMN_COMPLETED BETWEEN ${filterCompletedRangeStart?:Long.MIN_VALUE} AND ${filterCompletedRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_COMPLETED, COLUMN_COMPLETED_TIMEZONE, filterCompletedRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterCompletedRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " // DAY RANGE if(filterStartDayRangeStart != 0 || filterStartDayRangeEnd != 0) - queryString += " AND ($COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong() + filterStartDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterStartDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today.plusDays(filterStartDayRangeStart.toLong()), today.plusDays(filterStartDayRangeEnd.toLong()))} " if(filterDueDayRangeStart != 0 || filterDueDayRangeEnd != 0) - queryString += " AND ($COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong() + filterDueDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterDueDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today.plusDays(filterDueDayRangeStart.toLong()), today.plusDays(filterDueDayRangeEnd.toLong()))} " if(filterCompletedDayRangeStart != 0 || filterCompletedDayRangeEnd != 0) - queryString += " AND ($COLUMN_COMPLETED BETWEEN ${DateTimeUtils.getTodayAsLong() + filterCompletedDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterCompletedDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_COMPLETED, COLUMN_COMPLETED_TIMEZONE, today.plusDays(filterCompletedDayRangeStart.toLong()), today.plusDays(filterCompletedDayRangeEnd.toLong()))} " //CLASSIFICATION if (searchClassification.isNotEmpty()) { @@ -581,7 +635,7 @@ data class ICal4List( if (searchSettingShowOneRecurEntryInFuture) { queryString += "AND ($VIEW_NAME_ICAL4LIST.$COLUMN_RECURID IS NULL " + "OR $VIEW_NAME_ICAL4LIST.$COLUMN_DTSTART <= " + - "(SELECT MIN(recurList.$COLUMN_DTSTART) FROM $TABLE_NAME_ICALOBJECT as recurList WHERE recurList.$COLUMN_UID = $VIEW_NAME_ICAL4LIST.$COLUMN_UID AND recurList.$COLUMN_RECURID IS NOT NULL AND recurList.$COLUMN_DTSTART >= ${DateTimeUtils.getTodayAsLong()} )) " + "(SELECT MIN(recurList.$COLUMN_DTSTART) FROM $TABLE_NAME_ICALOBJECT as recurList WHERE recurList.$COLUMN_UID = $VIEW_NAME_ICAL4LIST.$COLUMN_UID AND recurList.$COLUMN_RECURID IS NOT NULL AND ${getDayRangeFilter("recurList.$COLUMN_DTSTART", "recurList.$COLUMN_DTSTART_TIMEZONE", today, null)} )) " } queryString += "ORDER BY $COLUMN_LAST_MODIFIED DESC " diff --git a/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt b/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt index ef4428793..447acd573 100644 --- a/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt +++ b/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt @@ -321,10 +321,39 @@ object DateTimeUtils { } } + /** + * jtx stores all-day dates as midnight UTC, while dates with a time are stored as the epoch + * milliseconds of the actual instant. Day based filters therefore need two different + * boundaries for one and the same day, this one is the boundary for all-day entries. + * + * @param date the day of which the beginning should be returned, today by default + * @return the beginning of the given day in UTC as epoch milliseconds + */ + fun getStartOfDayUTCAsLong(date: LocalDate = LocalDate.now()): Long = + date.atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + + /** + * Counterpart of [getStartOfDayUTCAsLong] for entries that are stored as a real instant + * (i.e. everything that is not an all-day entry). + * + * @param date the day of which the beginning should be returned, today by default + * @return the beginning of the given day in the local timezone as epoch milliseconds + */ + fun getStartOfDayLocalAsLong(date: LocalDate = LocalDate.now()): Long = + date.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli() + + /** + * @param utcMidnight a value that was stored as midnight UTC (all-day entries as well as the + * values that are delivered by the date (range) pickers) + * @return the [LocalDate] that this value represents + */ + fun getLocalDateFromUTCMidnight(utcMidnight: Long): LocalDate = + Instant.ofEpochMilli(utcMidnight).atZone(ZoneId.of("UTC")).toLocalDate() + /** * @return the current day as Long (the hour, minute, second and millisecond of the current datetime is set to 0) */ - fun getTodayAsLong() = LocalDate.now().atStartOfDay().atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() + fun getTodayAsLong() = getStartOfDayUTCAsLong() /** * @param datetime as Long diff --git a/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt b/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt new file mode 100644 index 000000000..9668f2dd9 --- /dev/null +++ b/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) Techbee e.U. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0 + * which accompanies this distribution, and is available at + * http://www.gnu.org/licenses/gpl.html + */ + +package at.techbee.jtx.database.views + +import at.techbee.jtx.database.COLUMN_DTSTART +import at.techbee.jtx.database.Module +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import java.time.LocalTime +import java.time.ZoneId +import java.util.TimeZone + +class ICal4ListTest { + + private val systemTimeZone = TimeZone.getDefault() + + /** a timezone with an offset that is not a full hour to also catch rounding to full hours */ + private val testTimeZone = TimeZone.getTimeZone("Australia/Adelaide") // +09:30/+10:30 + + @After + fun tearDown() { + TimeZone.setDefault(systemTimeZone) + } + + /** + * @return the range (from inclusive, until exclusive) that the given query applies to entries + * that are NOT all-day entries (the ELSE branch of the CASE), or null if it is not present + */ + private fun getRangeForEntriesWithTime(sql: String): LongRange? { + val match = Regex("ELSE $COLUMN_DTSTART >= (\\d+) AND $COLUMN_DTSTART < (\\d+) END").find(sql) ?: return null + return match.groupValues[1].toLong() until match.groupValues[2].toLong() + } + + private fun queryForStartToday(): String { + TimeZone.setDefault(testTimeZone) + return ICal4List.constructQuery( + modules = listOf(Module.TODO), + isFilterStartToday = true, + hideBiometricProtected = emptyList() + ).sql + } + + private fun asEpochMilli(date: LocalDate, time: LocalTime) = + date.atTime(time).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() + + @Test + fun constructQuery_filterStartToday_includesEntryLateToday() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertTrue(asEpochMilli(LocalDate.now(), LocalTime.of(23, 30)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_includesEntryEarlyToday() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertTrue(asEpochMilli(LocalDate.now(), LocalTime.of(0, 30)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_excludesEntryOfTomorrowMorning() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertFalse(asEpochMilli(LocalDate.now().plusDays(1), LocalTime.of(9, 0)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_excludesEntryOfYesterdayEvening() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertFalse(asEpochMilli(LocalDate.now().minusDays(1), LocalTime.of(22, 0)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_allDayEntriesAreComparedInUTC() { + val sql = queryForStartToday() + val todayAsAllDay = LocalDate.now().atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + val tomorrowAsAllDay = LocalDate.now().plusDays(1).atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + assertTrue(sql.contains("THEN $COLUMN_DTSTART >= $todayAsAllDay AND $COLUMN_DTSTART < $tomorrowAsAllDay ")) + } +} diff --git a/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt b/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt index 0e9a4e54c..bdc2d955f 100644 --- a/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt +++ b/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt @@ -16,6 +16,8 @@ import at.techbee.jtx.util.DateTimeUtils.getLongListfromCSVString import at.techbee.jtx.util.DateTimeUtils.getTodayAsLong import org.junit.Assert.assertEquals import org.junit.Test +import java.time.LocalDate +import java.util.TimeZone class DateTimeUtilsTest { @@ -80,6 +82,25 @@ class DateTimeUtilsTest { //@Test fun getDateWithoutTime_test_TZ_UTC() = assertEquals(1642550400000, getDateWithoutTime(1642590117816, "UTC")) //@Test fun getDateWithoutTime_TZ_Vienna() = assertEquals(1642546800000, getDateWithoutTime(1642590117816, "Europe/Vienna")) + @Test fun getStartOfDayUTCAsLong_isMidnightUTC() { + val startOfDay = DateTimeUtils.getStartOfDayUTCAsLong(LocalDate.of(2024, 3, 15)) + assertEquals(1710460800000L, startOfDay) // 2024-03-15T00:00:00Z + } + + @Test fun getStartOfDayLocalAsLong_isMidnightInTheLocalTimezone() { + val defaultTimeZone = TimeZone.getDefault() + try { + TimeZone.setDefault(TimeZone.getTimeZone("Australia/Adelaide")) // +10:30 on that day + val startOfDay = DateTimeUtils.getStartOfDayLocalAsLong(LocalDate.of(2024, 3, 15)) + assertEquals(1710460800000L - (10.5 * 60 * 60 * 1000).toLong(), startOfDay) + } finally { + TimeZone.setDefault(defaultTimeZone) + } + } + + @Test fun getLocalDateFromUTCMidnight_test() = + assertEquals(LocalDate.of(2024, 3, 15), DateTimeUtils.getLocalDateFromUTCMidnight(1710460800000L)) + @Test fun getMinutesSecondsFormatted_seconds_only_single() = assertEquals("00:05", DateTimeUtils.getMinutesSecondsFormatted(5)) @Test fun getMinutesSecondsFormatted_seconds_only_double() = assertEquals("00:55", DateTimeUtils.getMinutesSecondsFormatted(55)) @Test fun getMinutesSecondsFormatted_minutes_single() = assertEquals("01:15", DateTimeUtils.getMinutesSecondsFormatted(75))