diff --git a/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt b/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt index cd8759456..a3336b4f2 100644 --- a/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt +++ b/app/src/main/java/at/techbee/jtx/database/views/ICal4List.kt @@ -65,6 +65,7 @@ import at.techbee.jtx.database.COLUMN_URL import at.techbee.jtx.database.Classification import at.techbee.jtx.database.Component import at.techbee.jtx.database.ICalCollection.Factory.LOCAL_ACCOUNT_TYPE +import at.techbee.jtx.database.ICalObject.Companion.TZ_ALLDAY import at.techbee.jtx.database.Module import at.techbee.jtx.database.Status import at.techbee.jtx.database.TABLE_NAME_COLLECTION @@ -92,8 +93,8 @@ import at.techbee.jtx.database.properties.TABLE_NAME_RELATEDTO import at.techbee.jtx.database.properties.TABLE_NAME_RESOURCE import at.techbee.jtx.ui.list.AnyAllNone import at.techbee.jtx.util.DateTimeUtils +import java.time.LocalDate import java.util.UUID -import kotlin.time.Duration.Companion.days const val VIEW_NAME_ICAL4LIST = "ical4list" @@ -309,6 +310,57 @@ data class ICal4List( isReadOnly = true ) + /** + * jtx stores all-day entries as midnight UTC, while entries with a time are stored as the + * epoch milliseconds of the actual instant. Filtering both with the same UTC based + * boundaries makes entries of the neighbouring day slip into the filter on every device + * that is not set to UTC, so the boundaries have to be calculated for each of the two + * cases separately. + * + * @param dateColumn the column with the date(-time) to filter, e.g. [COLUMN_DTSTART] + * @param timezoneColumn the column with the timezone of [dateColumn], e.g. [COLUMN_DTSTART_TIMEZONE] + * @param from the first day of the filtered range (inclusive) or null for no lower boundary + * @param until the last day of the filtered range (inclusive) or null for no upper boundary + * @return the condition to add to the query + */ + private fun getDayRangeFilter( + dateColumn: String, + timezoneColumn: String, + from: LocalDate?, + until: LocalDate? + ): String { + fun boundaries(startOfDay: (LocalDate) -> Long): String { + val conditions = mutableListOf() + from?.let { conditions.add("$dateColumn >= ${startOfDay(it)}") } + until?.let { conditions.add("$dateColumn < ${startOfDay(it.plusDays(1))}") } + return if (conditions.isEmpty()) "$dateColumn IS NOT NULL" else conditions.joinToString(separator = " AND ") + } + + return "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN ${boundaries(DateTimeUtils::getStartOfDayUTCAsLong)} " + + "ELSE ${boundaries(DateTimeUtils::getStartOfDayLocalAsLong)} END)" + } + + /** + * All-day entries have no time, they are therefore compared against the beginning of today + * (in UTC, see [getDayRangeFilter]) instead of the current time. An all-day entry of today + * is consequently neither in the past nor in the future. + * + * @return the condition that the given date(-time) lies before now + */ + private fun getBeforeNowFilter(dateColumn: String, timezoneColumn: String) = + "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN $dateColumn < ${DateTimeUtils.getStartOfDayUTCAsLong()} " + + "ELSE $dateColumn < ${System.currentTimeMillis()} END)" + + /** + * @return the condition that the given date(-time) lies after now, see [getBeforeNowFilter] + */ + private fun getAfterNowFilter(dateColumn: String, timezoneColumn: String) = + "(CASE WHEN $timezoneColumn = '$TZ_ALLDAY' " + + "THEN $dateColumn >= ${DateTimeUtils.getStartOfDayUTCAsLong(LocalDate.now().plusDays(1))} " + + "ELSE $dateColumn > ${System.currentTimeMillis()} END)" + fun constructQuery( modules: List, searchCategories: List = emptyList(), @@ -470,27 +522,29 @@ data class ICal4List( if (isExcludeDone) queryString += "AND $COLUMN_PERCENT IS NOT 100 AND ($COLUMN_STATUS IS NULL OR $COLUMN_STATUS NOT IN ('${Status.COMPLETED.status}', '${Status.CANCELLED.status}')) " + val today = LocalDate.now() + val dateQuery = mutableListOf() if (isFilterStartInPast) - dateQuery.add("$COLUMN_DTSTART < ${System.currentTimeMillis()}") + dateQuery.add(getBeforeNowFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE)) if (isFilterStartToday) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today, today)) if (isFilterStartTomorrow) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (2).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today.plusDays(1), today.plusDays(1))) if (isFilterStartWithin7Days) - dateQuery.add("$COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (8).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today, today.plusDays(7))) if (isFilterStartFuture) - dateQuery.add("$COLUMN_DTSTART > ${System.currentTimeMillis()}") + dateQuery.add(getAfterNowFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE)) if (isFilterOverdue) - dateQuery.add("$COLUMN_DUE < ${System.currentTimeMillis()}") + dateQuery.add(getBeforeNowFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE)) if (isFilterDueToday) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today, today)) if (isFilterDueTomorrow) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong() + (1).days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (2).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today.plusDays(1), today.plusDays(1))) if (isFilterDueWithin7Days) - dateQuery.add("$COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong()} AND ${DateTimeUtils.getTodayAsLong() + (8).days.inWholeMilliseconds - 1}") + dateQuery.add(getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today, today.plusDays(7))) if (isFilterDueFuture) - dateQuery.add("$COLUMN_DUE > ${System.currentTimeMillis()}") + dateQuery.add(getAfterNowFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE)) if (isFilterNoDatesSet) dateQuery.add("$COLUMN_DTSTART IS NULL AND $COLUMN_DUE IS NULL AND $COLUMN_COMPLETED IS NULL ") if (isFilterNoStartDateSet) @@ -503,21 +557,21 @@ data class ICal4List( if (dateQuery.isNotEmpty()) queryString += " AND (${dateQuery.joinToString(separator = " OR ")}) " - // DATE RANGE + // DATE RANGE (the date pickers deliver the selected days as midnight UTC) if(filterStartRangeStart != null || filterStartRangeEnd != null) - queryString += " AND ($COLUMN_DTSTART BETWEEN ${filterStartRangeStart?:Long.MIN_VALUE} AND ${filterStartRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, filterStartRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterStartRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " if(filterDueRangeStart != null || filterDueRangeEnd != null) - queryString += " AND ($COLUMN_DUE BETWEEN ${filterDueRangeStart?:Long.MIN_VALUE} AND ${filterDueRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, filterDueRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterDueRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " if(filterCompletedRangeStart != null || filterCompletedRangeEnd != null) - queryString += " AND ($COLUMN_COMPLETED BETWEEN ${filterCompletedRangeStart?:Long.MIN_VALUE} AND ${filterCompletedRangeEnd?.let { it + (1).days.inWholeMilliseconds-1 }?:Long.MAX_VALUE})" + queryString += " AND ${getDayRangeFilter(COLUMN_COMPLETED, COLUMN_COMPLETED_TIMEZONE, filterCompletedRangeStart?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) }, filterCompletedRangeEnd?.let { DateTimeUtils.getLocalDateFromUTCMidnight(it) })} " // DAY RANGE if(filterStartDayRangeStart != 0 || filterStartDayRangeEnd != 0) - queryString += " AND ($COLUMN_DTSTART BETWEEN ${DateTimeUtils.getTodayAsLong() + filterStartDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterStartDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_DTSTART, COLUMN_DTSTART_TIMEZONE, today.plusDays(filterStartDayRangeStart.toLong()), today.plusDays(filterStartDayRangeEnd.toLong()))} " if(filterDueDayRangeStart != 0 || filterDueDayRangeEnd != 0) - queryString += " AND ($COLUMN_DUE BETWEEN ${DateTimeUtils.getTodayAsLong() + filterDueDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterDueDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_DUE, COLUMN_DUE_TIMEZONE, today.plusDays(filterDueDayRangeStart.toLong()), today.plusDays(filterDueDayRangeEnd.toLong()))} " if(filterCompletedDayRangeStart != 0 || filterCompletedDayRangeEnd != 0) - queryString += " AND ($COLUMN_COMPLETED BETWEEN ${DateTimeUtils.getTodayAsLong() + filterCompletedDayRangeStart.days.inWholeMilliseconds} AND ${DateTimeUtils.getTodayAsLong() + (filterCompletedDayRangeEnd+1).days.inWholeMilliseconds - 1})" + queryString += " AND ${getDayRangeFilter(COLUMN_COMPLETED, COLUMN_COMPLETED_TIMEZONE, today.plusDays(filterCompletedDayRangeStart.toLong()), today.plusDays(filterCompletedDayRangeEnd.toLong()))} " //CLASSIFICATION if (searchClassification.isNotEmpty()) { @@ -581,7 +635,7 @@ data class ICal4List( if (searchSettingShowOneRecurEntryInFuture) { queryString += "AND ($VIEW_NAME_ICAL4LIST.$COLUMN_RECURID IS NULL " + "OR $VIEW_NAME_ICAL4LIST.$COLUMN_DTSTART <= " + - "(SELECT MIN(recurList.$COLUMN_DTSTART) FROM $TABLE_NAME_ICALOBJECT as recurList WHERE recurList.$COLUMN_UID = $VIEW_NAME_ICAL4LIST.$COLUMN_UID AND recurList.$COLUMN_RECURID IS NOT NULL AND recurList.$COLUMN_DTSTART >= ${DateTimeUtils.getTodayAsLong()} )) " + "(SELECT MIN(recurList.$COLUMN_DTSTART) FROM $TABLE_NAME_ICALOBJECT as recurList WHERE recurList.$COLUMN_UID = $VIEW_NAME_ICAL4LIST.$COLUMN_UID AND recurList.$COLUMN_RECURID IS NOT NULL AND ${getDayRangeFilter("recurList.$COLUMN_DTSTART", "recurList.$COLUMN_DTSTART_TIMEZONE", today, null)} )) " } queryString += "ORDER BY $COLUMN_LAST_MODIFIED DESC " diff --git a/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt b/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt index ef4428793..447acd573 100644 --- a/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt +++ b/app/src/main/java/at/techbee/jtx/util/DateTimeUtils.kt @@ -321,10 +321,39 @@ object DateTimeUtils { } } + /** + * jtx stores all-day dates as midnight UTC, while dates with a time are stored as the epoch + * milliseconds of the actual instant. Day based filters therefore need two different + * boundaries for one and the same day, this one is the boundary for all-day entries. + * + * @param date the day of which the beginning should be returned, today by default + * @return the beginning of the given day in UTC as epoch milliseconds + */ + fun getStartOfDayUTCAsLong(date: LocalDate = LocalDate.now()): Long = + date.atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + + /** + * Counterpart of [getStartOfDayUTCAsLong] for entries that are stored as a real instant + * (i.e. everything that is not an all-day entry). + * + * @param date the day of which the beginning should be returned, today by default + * @return the beginning of the given day in the local timezone as epoch milliseconds + */ + fun getStartOfDayLocalAsLong(date: LocalDate = LocalDate.now()): Long = + date.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli() + + /** + * @param utcMidnight a value that was stored as midnight UTC (all-day entries as well as the + * values that are delivered by the date (range) pickers) + * @return the [LocalDate] that this value represents + */ + fun getLocalDateFromUTCMidnight(utcMidnight: Long): LocalDate = + Instant.ofEpochMilli(utcMidnight).atZone(ZoneId.of("UTC")).toLocalDate() + /** * @return the current day as Long (the hour, minute, second and millisecond of the current datetime is set to 0) */ - fun getTodayAsLong() = LocalDate.now().atStartOfDay().atZone(ZoneId.of("UTC")).toInstant().toEpochMilli() + fun getTodayAsLong() = getStartOfDayUTCAsLong() /** * @param datetime as Long diff --git a/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt b/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt new file mode 100644 index 000000000..9668f2dd9 --- /dev/null +++ b/app/src/test/java/at/techbee/jtx/database/views/ICal4ListTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) Techbee e.U. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the GNU Public License v3.0 + * which accompanies this distribution, and is available at + * http://www.gnu.org/licenses/gpl.html + */ + +package at.techbee.jtx.database.views + +import at.techbee.jtx.database.COLUMN_DTSTART +import at.techbee.jtx.database.Module +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.time.LocalDate +import java.time.LocalTime +import java.time.ZoneId +import java.util.TimeZone + +class ICal4ListTest { + + private val systemTimeZone = TimeZone.getDefault() + + /** a timezone with an offset that is not a full hour to also catch rounding to full hours */ + private val testTimeZone = TimeZone.getTimeZone("Australia/Adelaide") // +09:30/+10:30 + + @After + fun tearDown() { + TimeZone.setDefault(systemTimeZone) + } + + /** + * @return the range (from inclusive, until exclusive) that the given query applies to entries + * that are NOT all-day entries (the ELSE branch of the CASE), or null if it is not present + */ + private fun getRangeForEntriesWithTime(sql: String): LongRange? { + val match = Regex("ELSE $COLUMN_DTSTART >= (\\d+) AND $COLUMN_DTSTART < (\\d+) END").find(sql) ?: return null + return match.groupValues[1].toLong() until match.groupValues[2].toLong() + } + + private fun queryForStartToday(): String { + TimeZone.setDefault(testTimeZone) + return ICal4List.constructQuery( + modules = listOf(Module.TODO), + isFilterStartToday = true, + hideBiometricProtected = emptyList() + ).sql + } + + private fun asEpochMilli(date: LocalDate, time: LocalTime) = + date.atTime(time).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli() + + @Test + fun constructQuery_filterStartToday_includesEntryLateToday() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertTrue(asEpochMilli(LocalDate.now(), LocalTime.of(23, 30)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_includesEntryEarlyToday() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertTrue(asEpochMilli(LocalDate.now(), LocalTime.of(0, 30)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_excludesEntryOfTomorrowMorning() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertFalse(asEpochMilli(LocalDate.now().plusDays(1), LocalTime.of(9, 0)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_excludesEntryOfYesterdayEvening() { + val range = getRangeForEntriesWithTime(queryForStartToday()) + assertNotNull(range) + assertFalse(asEpochMilli(LocalDate.now().minusDays(1), LocalTime.of(22, 0)) in range!!) + } + + @Test + fun constructQuery_filterStartToday_allDayEntriesAreComparedInUTC() { + val sql = queryForStartToday() + val todayAsAllDay = LocalDate.now().atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + val tomorrowAsAllDay = LocalDate.now().plusDays(1).atStartOfDay(ZoneId.of("UTC")).toInstant().toEpochMilli() + assertTrue(sql.contains("THEN $COLUMN_DTSTART >= $todayAsAllDay AND $COLUMN_DTSTART < $tomorrowAsAllDay ")) + } +} diff --git a/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt b/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt index 0e9a4e54c..bdc2d955f 100644 --- a/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt +++ b/app/src/test/java/at/techbee/jtx/util/DateTimeUtilsTest.kt @@ -16,6 +16,8 @@ import at.techbee.jtx.util.DateTimeUtils.getLongListfromCSVString import at.techbee.jtx.util.DateTimeUtils.getTodayAsLong import org.junit.Assert.assertEquals import org.junit.Test +import java.time.LocalDate +import java.util.TimeZone class DateTimeUtilsTest { @@ -80,6 +82,25 @@ class DateTimeUtilsTest { //@Test fun getDateWithoutTime_test_TZ_UTC() = assertEquals(1642550400000, getDateWithoutTime(1642590117816, "UTC")) //@Test fun getDateWithoutTime_TZ_Vienna() = assertEquals(1642546800000, getDateWithoutTime(1642590117816, "Europe/Vienna")) + @Test fun getStartOfDayUTCAsLong_isMidnightUTC() { + val startOfDay = DateTimeUtils.getStartOfDayUTCAsLong(LocalDate.of(2024, 3, 15)) + assertEquals(1710460800000L, startOfDay) // 2024-03-15T00:00:00Z + } + + @Test fun getStartOfDayLocalAsLong_isMidnightInTheLocalTimezone() { + val defaultTimeZone = TimeZone.getDefault() + try { + TimeZone.setDefault(TimeZone.getTimeZone("Australia/Adelaide")) // +10:30 on that day + val startOfDay = DateTimeUtils.getStartOfDayLocalAsLong(LocalDate.of(2024, 3, 15)) + assertEquals(1710460800000L - (10.5 * 60 * 60 * 1000).toLong(), startOfDay) + } finally { + TimeZone.setDefault(defaultTimeZone) + } + } + + @Test fun getLocalDateFromUTCMidnight_test() = + assertEquals(LocalDate.of(2024, 3, 15), DateTimeUtils.getLocalDateFromUTCMidnight(1710460800000L)) + @Test fun getMinutesSecondsFormatted_seconds_only_single() = assertEquals("00:05", DateTimeUtils.getMinutesSecondsFormatted(5)) @Test fun getMinutesSecondsFormatted_seconds_only_double() = assertEquals("00:55", DateTimeUtils.getMinutesSecondsFormatted(55)) @Test fun getMinutesSecondsFormatted_minutes_single() = assertEquals("01:15", DateTimeUtils.getMinutesSecondsFormatted(75))