From 9dc27a1688c1edfd65436ca477d72095a5d3620f Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 9 Jul 2026 00:30:55 +0800 Subject: [PATCH 1/4] feat: global temporal and leave-last-out splitting for next-item evaluation --- cornac/data/dataset.py | 14 + cornac/eval_methods/next_item_evaluation.py | 301 ++++++++++++++++++ tests/cornac/data/test_dataset.py | 57 ++++ .../eval_methods/test_next_item_evaluation.py | 171 ++++++++++ 4 files changed, 543 insertions(+) diff --git a/cornac/data/dataset.py b/cornac/data/dataset.py index a5629c9cb..6face7aa2 100644 --- a/cornac/data/dataset.py +++ b/cornac/data/dataset.py @@ -1203,6 +1203,20 @@ def build( extra_pos = ts_pos + 1 extra_data = [data[i][extra_pos] for i in valid_idx] if fmt in ["SITJson", "USITJson"] else None + if timestamps is not None and len(timestamps) > 1: + order = np.argsort(session_indices, kind="stable") + s = session_indices[order] + t = timestamps[order] + decreasing = (t[1:] < t[:-1]) & (s[1:] == s[:-1]) + if decreasing.any(): + n_bad = int(decreasing.sum()) + warnings.warn( + f"{n_bad} interaction(s) are not in chronological order within " + "their session. Sequential models treat input row order as the " + "ground-truth sequence; sort your data by (session, timestamp) " + "before building the dataset." + ) + dataset = cls( num_users=len(global_uid_map), num_sessions=len(set(session_indices)), diff --git a/cornac/eval_methods/next_item_evaluation.py b/cornac/eval_methods/next_item_evaluation.py index 9d8ee4e1a..e7cedc02d 100644 --- a/cornac/eval_methods/next_item_evaluation.py +++ b/cornac/eval_methods/next_item_evaluation.py @@ -23,6 +23,7 @@ from ..data import SequentialDataset from ..experiment.result import Result from ..models import NextItemRecommender +from ..utils import validate_format from . import BaseMethod EVALUATION_MODES = frozenset([ @@ -189,6 +190,19 @@ class NextItemEvaluation(BaseMethod): verbose: bool, optional, default: False Output running log. + Notes + ----- + **Data splitting.** Ratio-based splitting (inherited from + :obj:`BaseMethod`) and per-user leave-last-out both leak future + information into training: a random split trains on interactions that + postdate the test items, while leave-last-out places each user's held-out + item at a different absolute time, so training on one user includes other + users' later interactions (popularity drift then distorts results). The + recommended protocol is a single global temporal cutoff; use + :meth:`from_timestamps` for a leakage-free, session-level split. + Per-user leave-last-out remains available via :meth:`leave_last_out` + for comparability with published results. + """ def __init__( @@ -469,3 +483,290 @@ def from_splits( test_data=test_data, val_data=val_data, ) + + @classmethod + def from_timestamps( + cls, + data, + test_timestamp, + val_timestamp=None, + fmt="USIT", + exclude_unknowns=True, + mode="last", + seed=None, + verbose=False, + **kwargs, + ): + """Constructing evaluation method by a global temporal split. + + Sessions are split by a single global time horizon so that no + training session is evaluated against future information shared across + users. This is the leakage-free protocol recommended by the + offline-evaluation literature (see Notes). + + Parameters + ---------- + data: list, required + Raw preference data in the tuple format given by `fmt`. + + test_timestamp: float, required + Absolute timestamp (same unit as the timestamps in `data`) marking + the start of the test period. Sessions whose last event is at or + after this timestamp form the test set. + + val_timestamp: float, optional, default: None + Absolute timestamp marking the start of the validation period. If + None, no validation set is created and the training set absorbs + everything before `test_timestamp`. Must be strictly smaller than + `test_timestamp`. + + fmt: str, default: 'USIT' + Format of the input data. Currently, we are supporting: + + 'SIT': Session, Item, Timestamp + 'USIT': User, Session, Item, Timestamp + 'SITJson': Session, Item, Timestamp, Json + 'USITJson': User, Session, Item, Timestamp, Json + + exclude_unknowns: bool, optional, default: True + Whether to exclude unknown users/items in evaluation. + + mode: str, optional, default: 'last' + Evaluation mode is either 'next' or 'last'. + If 'last', only evaluate the last item. + If 'next', evaluate every next item in the sequence. + + seed: int, optional, default: None + Random seed for reproducibility. + + verbose: bool, optional, default: False + The verbosity flag. + + Returns + ------- + method: :obj:`` + Evaluation method object. + + Notes + ----- + Sessions are atomic: each session is assigned to exactly one split by + the timestamp of its **last event** ``last_ts = max(t of session)``: + + - train: ``last_ts < val_timestamp`` + - val: ``val_timestamp <= last_ts < test_timestamp`` + - test: ``last_ts >= test_timestamp`` + + ``>=`` goes to the later split (same convention as + :obj:`cornac.eval_methods.TimestampSplit`). With + ``val_timestamp=None``, train is ``last_ts < test_timestamp``. + + Assigning whole sessions by last event keeps sessions intact (an + interaction-level cutoff would truncate straddling sessions), bounding + residual leakage by session length (Hidasi & Czapp, RecSys 2023). + Rows keep their original relative order within each partition, as + required by :obj:`cornac.data.SequentialDataset.build`. + + References + ---------- + Meng et al. (2020). Exploring Data Splitting Strategies for the + Evaluation of Recommendation Models. RecSys 2020. + + Ji et al. (2023). A Critical Study on Data Leakage in Recommender + System Offline Evaluation. ACM TOIS 2023. + + Hidasi & Czapp (2023). Widespread Flaws in Offline Evaluation of + Recommender Systems. RecSys 2023. + + """ + fmt = validate_format(fmt, ["SIT", "USIT", "SITJson", "USITJson"]) + + if val_timestamp is not None and val_timestamp >= test_timestamp: + raise ValueError( + "val_timestamp ({}) must be strictly smaller than " + "test_timestamp ({}).".format(val_timestamp, test_timestamp) + ) + + sid_pos = 1 if fmt in ["USIT", "USITJson"] else 0 + ts_pos = 3 if fmt in ["USIT", "USITJson"] else 2 + + # Pass 1: last event timestamp per session. + last_ts = {} + for tup in data: + sid = tup[sid_pos] + t = float(tup[ts_pos]) + if sid not in last_ts or t > last_ts[sid]: + last_ts[sid] = t + + # Pass 2: route tuples, preserving original relative order. + train_data, val_data, test_data = [], [], [] + for tup in data: + ts = last_ts[tup[sid_pos]] + if ts >= test_timestamp: + test_data.append(tup) + elif val_timestamp is not None and ts >= val_timestamp: + val_data.append(tup) + else: + train_data.append(tup) + + if len(train_data) == 0: + raise ValueError( + "Empty train partition: no session ends before the cutoff " + "({}).".format(test_timestamp if val_timestamp is None else val_timestamp) + ) + if len(test_data) == 0: + raise ValueError( + "Empty test partition: no session ends at or after " + "test_timestamp ({}).".format(test_timestamp) + ) + if val_timestamp is not None and len(val_data) == 0: + warnings.warn( + "Empty validation partition: no session ends in " + "[{}, {}). Proceeding with no validation set.".format( + val_timestamp, test_timestamp + ) + ) + val_data = None + + if verbose: + def _n_sessions(part): + return len({tup[sid_pos] for tup in part}) if part else 0 + + print("---") + print("Global temporal split:") + print( + "Train: {} sessions, {} interactions".format( + _n_sessions(train_data), len(train_data) + ) + ) + print( + "Val: {} sessions, {} interactions".format( + _n_sessions(val_data), len(val_data) if val_data else 0 + ) + ) + print( + "Test: {} sessions, {} interactions".format( + _n_sessions(test_data), len(test_data) + ) + ) + + return cls.from_splits( + train_data=train_data, + test_data=test_data, + val_data=val_data, + fmt=fmt, + exclude_unknowns=exclude_unknowns, + seed=seed, + verbose=verbose, + mode=mode, + **kwargs, + ) + + @classmethod + def leave_last_out( + cls, + data, + fmt="UIRT", + exclude_unknowns=True, + mode="last", + seed=None, + verbose=False, + **kwargs, + ): + """Constructing evaluation method by per-user leave-last-out. + + Each user's interactions are sorted chronologically and treated as one + session (session id = user id). The last item is held out for testing + and the second-to-last for validation — the common protocol in the + sequential recommendation literature (SASRec, BERT4Rec, ...). + + Parameters + ---------- + data: list, required + Raw preference data in the quadruplet format + [(user_id, item_id, rating, timestamp)]. Ratings are ignored + (implicit next-item feedback). + + fmt: str, default: 'UIRT' + Format of the input data. Only 'UIRT' is supported. + + exclude_unknowns: bool, optional, default: True + Whether to exclude unknown users/items in evaluation. + + mode: str, optional, default: 'last' + Evaluation mode is either 'next' or 'last'. + If 'last', only evaluate the last item. + If 'next', evaluate every next item in the sequence. + + seed: int, optional, default: None + Random seed for reproducibility. + + verbose: bool, optional, default: False + The verbosity flag. + + Returns + ------- + method: :obj:`` + Evaluation method object. + + Notes + ----- + Per user (chronological, stable sort — tied timestamps keep input + order), with cumulative sequences: + + - train: ``seq[:-2]`` + - val: ``seq[:-1]`` (target = second-to-last item) + - test: ``seq`` (target = last item) + + Users with fewer than 3 interactions are dropped from all splits. + + This protocol leaks future information across users: each held-out + item sits at a different absolute time, so training includes other + users' later interactions (Ji et al., A Critical Study on Data + Leakage in Recommender System Offline Evaluation, ACM TOIS 2023). + It is provided for comparability with published results; prefer + :meth:`from_timestamps` for a leakage-free protocol. + + """ + fmt = validate_format(fmt, ["UIRT"]) + + by_user = OrderedDict() + for u, i, _, t in data: + by_user.setdefault(u, []).append((float(t), i, t)) + + train_data, val_data, test_data = [], [], [] + n_skipped = 0 + for u, events in by_user.items(): + if len(events) < 3: + n_skipped += 1 + continue + events.sort(key=lambda x: x[0]) + seq = [(u, u, i, t) for _, i, t in events] + train_data.extend(seq[:-2]) + val_data.extend(seq[:-1]) + test_data.extend(seq) + + if len(train_data) == 0: + raise ValueError( + "Empty train set: no user has at least 3 interactions." + ) + + if verbose: + print("---") + print("Leave-last-out split (user = session):") + print( + "{} users kept, {} users with < 3 interactions dropped".format( + len(by_user) - n_skipped, n_skipped + ) + ) + + return cls.from_splits( + train_data=train_data, + test_data=test_data, + val_data=val_data, + fmt="USIT", + exclude_unknowns=exclude_unknowns, + seed=seed, + verbose=verbose, + mode=mode, + **kwargs, + ) diff --git a/tests/cornac/data/test_dataset.py b/tests/cornac/data/test_dataset.py index c8458ec41..2ab9e276c 100644 --- a/tests/cornac/data/test_dataset.py +++ b/tests/cornac/data/test_dataset.py @@ -14,6 +14,7 @@ # ============================================================================ import unittest +import warnings import numpy as np import numpy.testing as npt @@ -295,6 +296,62 @@ def test_init(self): set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]), ) + +class TestSequentialDatasetSortOrder(unittest.TestCase): + def _assert_no_warning(self, data): + with warnings.catch_warnings(): + warnings.simplefilter("error") + SequentialDataset.from_sit(data) + + def test_shuffled_within_session_warns(self): + data = [("s1", "a", 3), ("s1", "b", 1), ("s1", "c", 2)] + with self.assertWarns(UserWarning): + SequentialDataset.from_sit(data) + + def test_per_session_sorted_no_warning(self): + data = [ + ("s1", "a", 1), + ("s1", "b", 2), + ("s2", "c", 1), + ("s2", "d", 2), + ] + self._assert_no_warning(data) + + def test_interleaved_sessions_sorted_no_warning(self): + # Rows of session A and B alternate; each session is internally sorted. + data = [ + ("A", "x", 1), + ("B", "p", 1), + ("A", "y", 2), + ("B", "q", 2), + ] + self._assert_no_warning(data) + + def test_noncontiguous_cross_block_disorder_warns(self): + # Session A appears in two blocks; the later A block is earlier in time. + data = [ + ("A", "x", 5), + ("A", "y", 6), + ("B", "p", 1), + ("A", "z", 2), + ] + with self.assertWarns(UserWarning): + SequentialDataset.from_sit(data) + + def test_tied_timestamps_no_warning(self): + data = [("A", "x", 1), ("A", "y", 1), ("A", "z", 1)] + self._assert_no_warning(data) + + def test_single_row_no_warning(self): + self._assert_no_warning([("A", "x", 1)]) + + def test_empty_data_no_sort_warning(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + with self.assertRaises(ValueError): + SequentialDataset.from_sit([]) + + class TestPurchaseViewDataset(unittest.TestCase): def test_build_extends_id_space(self): # User "u2" and item "i3" appear only in the view stream; they must diff --git a/tests/cornac/eval_methods/test_next_item_evaluation.py b/tests/cornac/eval_methods/test_next_item_evaluation.py index 31ceb8d9b..b0c85ab91 100644 --- a/tests/cornac/eval_methods/test_next_item_evaluation.py +++ b/tests/cornac/eval_methods/test_next_item_evaluation.py @@ -14,6 +14,7 @@ # ============================================================================ import unittest +import warnings from cornac.eval_methods import NextItemEvaluation from cornac.data import Reader @@ -21,6 +22,12 @@ from cornac.metrics import HitRatio, Recall +def _split_sids(dataset): + """Raw session ids actually present in a built split.""" + inv = {v: k for k, v in dataset.sid_map.items()} + return {inv[i] for i in dataset.sessions.keys()} + + class TestNextItemEvaluation(unittest.TestCase): def setUp(self): self.data = Reader().read("./tests/sequence.txt", fmt="USIT", sep=" ") @@ -63,5 +70,169 @@ def test_evaluate(self): self.assertEqual(result[0].metric_avg_results.get('HitRatio@5'), 3/4) self.assertEqual(result[0].metric_avg_results.get('Recall@5'), 3/4) +class TestFromTimestamps(unittest.TestCase): + def setUp(self): + # USIT: (user, session, item, timestamp). Rows interleaved across + # sessions but chronological within each session. Session last-event + # timestamps: s1=10, s2=40, s3=50, s4=70, s5=100, s6=130. + self.usit = [ + ("u1", "s1", "a", 5), + ("u1", "s1", "b", 10), + ("u2", "s2", "a", 20), + ("u2", "s2", "c", 40), + ("u1", "s3", "b", 45), + ("u1", "s3", "d", 50), + ("u3", "s4", "a", 30), + ("u3", "s4", "e", 70), + ("u2", "s5", "c", 80), # straddles test_timestamp=100 ... + ("u2", "s5", "b", 100), # ... but last event decides -> test + ("u4", "s6", "f", 120), + ("u4", "s6", "a", 130), + ] + + def test_toy_assignment(self): + m = NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=100, val_timestamp=50, fmt="USIT", + ) + self.assertEqual(m.train_set.num_sessions, 2) + self.assertEqual(m.val_set.num_sessions, 2) + self.assertEqual(m.test_set.num_sessions, 2) + self.assertEqual(_split_sids(m.train_set), {"s1", "s2"}) + self.assertEqual(_split_sids(m.val_set), {"s3", "s4"}) + # s5 straddles the cutoff (event at 80) yet lands in test by its last + # event at 100. + self.assertEqual(_split_sids(m.test_set), {"s5", "s6"}) + + def test_boundary_equality(self): + m = NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=100, val_timestamp=50, fmt="USIT", + ) + # s3 last_ts == val_timestamp -> val; s5 last_ts == test_timestamp -> test + self.assertIn("s3", _split_sids(m.val_set)) + self.assertIn("s5", _split_sids(m.test_set)) + + def test_no_validation(self): + m = NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=100, val_timestamp=None, fmt="USIT", + ) + self.assertIsNone(m.val_set) + # train absorbs the middle sessions (last_ts < 100) + self.assertEqual(_split_sids(m.train_set), {"s1", "s2", "s3", "s4"}) + self.assertEqual(_split_sids(m.test_set), {"s5", "s6"}) + + def test_val_ge_test_raises(self): + with self.assertRaises(ValueError): + NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=100, val_timestamp=100, fmt="USIT", + ) + + def test_empty_test_raises(self): + with self.assertRaises(ValueError): + NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=1000, val_timestamp=None, fmt="USIT", + ) + + def test_sit_format(self): + # SIT: (session, item, timestamp) -- no user column. + sit = [ + ("s1", "a", 5), + ("s1", "b", 10), + ("s2", "a", 20), + ("s2", "c", 40), + ("s3", "c", 80), + ("s3", "b", 100), + ] + m = NextItemEvaluation.from_timestamps( + sit, test_timestamp=100, val_timestamp=None, fmt="SIT", + ) + self.assertEqual(_split_sids(m.train_set), {"s1", "s2"}) + self.assertEqual(_split_sids(m.test_set), {"s3"}) + + def test_sorted_input_no_warning(self): + # Chronological-within-session input must not trigger the task-01 + # sort-order warning from SequentialDataset.build. + with warnings.catch_warnings(): + warnings.simplefilter("error") + NextItemEvaluation.from_timestamps( + self.usit, test_timestamp=100, val_timestamp=50, fmt="USIT", + ) + + +class TestLeaveLastOut(unittest.TestCase): + def setUp(self): + # UIRT: (user, item, rating, timestamp). Rows deliberately unsorted + # within users. u1 has 4 interactions, u2 has 3, u3 only 2 (dropped). + self.uirt = [ + ("u1", "c", 1.0, 30), + ("u1", "a", 1.0, 10), + ("u2", "b", 1.0, 25), + ("u1", "d", 1.0, 40), + ("u2", "a", 1.0, 5), + ("u1", "b", 1.0, 20), + ("u2", "c", 1.0, 45), + ("u3", "a", 1.0, 15), + ("u3", "b", 1.0, 35), + ] + + def test_split_sizes_and_users(self): + # exclude_unknowns=False isolates the split mechanics from the + # unknown-item filtering done by from_splits. + m = NextItemEvaluation.leave_last_out(self.uirt, exclude_unknowns=False) + # kept users: u1 (n=4), u2 (n=3); u3 dropped (< 3 interactions). + # cumulative splits: train sum(n-2)=3, val sum(n-1)=5, test sum(n)=7 + self.assertEqual(len(m.train_set.uir_tuple[0]), 3) + self.assertEqual(len(m.val_set.uir_tuple[0]), 5) + self.assertEqual(len(m.test_set.uir_tuple[0]), 7) + self.assertEqual(_split_sids(m.train_set), {"u1", "u2"}) + self.assertEqual(_split_sids(m.val_set), {"u1", "u2"}) + self.assertEqual(_split_sids(m.test_set), {"u1", "u2"}) + + def test_exclude_unknowns_default(self): + # With the default exclude_unknowns=True, val/test items that never + # appear in train (each user's held-out tail) are filtered out: + # train has {a, b}; val keeps 4 of 5 rows, test keeps 4 of 7. + m = NextItemEvaluation.leave_last_out(self.uirt) + self.assertEqual(len(m.train_set.uir_tuple[0]), 3) + self.assertEqual(len(m.val_set.uir_tuple[0]), 4) + self.assertEqual(len(m.test_set.uir_tuple[0]), 4) + + def test_chronological_item_order(self): + m = NextItemEvaluation.leave_last_out(self.uirt, exclude_unknowns=False) + # u1's test session must be [a, b, c, d] (sorted by time, not input). + test = m.test_set + inv_iid = {v: k for k, v in test.iid_map.items()} + sid = test.sid_map["u1"] + items = [inv_iid[test.uir_tuple[1][idx]] for idx in test.sessions[sid]] + self.assertEqual(items, ["a", "b", "c", "d"]) + + def test_too_few_interactions_raises(self): + with self.assertRaises(ValueError): + NextItemEvaluation.leave_last_out(self.uirt[-2:]) # only u3 + + def test_tied_timestamps_stable(self): + # a and b share t=10: input order must be preserved on ties. + uirt = [ + ("u1", "a", 1.0, 10), + ("u1", "b", 1.0, 10), + ("u1", "c", 1.0, 20), + ] + m = NextItemEvaluation.leave_last_out(uirt, exclude_unknowns=False) + test = m.test_set + inv_iid = {v: k for k, v in test.iid_map.items()} + sid = test.sid_map["u1"] + items = [inv_iid[test.uir_tuple[1][idx]] for idx in test.sessions[sid]] + self.assertEqual(items, ["a", "b", "c"]) + + def test_no_warning(self): + # Output is sorted per session; the task-01 warning must not fire. + with warnings.catch_warnings(): + warnings.simplefilter("error") + NextItemEvaluation.leave_last_out(self.uirt) + + def test_mode_passthrough(self): + m = NextItemEvaluation.leave_last_out(self.uirt, mode="next") + self.assertEqual(m.mode, "next") + + if __name__ == "__main__": unittest.main() From 4a5fba1f1ed5d0961e2c4b15cb8835bbe938c89a Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 9 Jul 2026 00:31:30 +0800 Subject: [PATCH 2/4] add Amazon Review dataset support --- cornac/datasets/__init__.py | 1 + cornac/datasets/amazon_review.py | 108 ++++++++++++++++++++ tests/cornac/datasets/test_amazon_review.py | 65 ++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 cornac/datasets/amazon_review.py create mode 100644 tests/cornac/datasets/test_amazon_review.py diff --git a/cornac/datasets/__init__.py b/cornac/datasets/__init__.py index 2f955bb69..02d79185d 100644 --- a/cornac/datasets/__init__.py +++ b/cornac/datasets/__init__.py @@ -16,6 +16,7 @@ from . import amazon_clothing from . import amazon_digital_music from . import amazon_office +from . import amazon_review from . import amazon_toy from . import citeulike from . import cosmetics diff --git a/cornac/datasets/amazon_review.py b/cornac/datasets/amazon_review.py new file mode 100644 index 000000000..487e765ca --- /dev/null +++ b/cornac/datasets/amazon_review.py @@ -0,0 +1,108 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Amazon Product Review datasets. + +There are three versions: '2014', '2018', and '2023' available. +'2014' is the version used in the Semantic-ID literature (e.g., TIGER). + +Source: https://cseweb.ucsd.edu/~jmcauley/datasets/amazon/links.html +""" + +import gzip +import json +import os +from typing import List + +from ..data import Reader +from ..utils import cache + +# category -> reviews__5.json.gz +_CATEGORY_FILES = { + "beauty": "Beauty", + "sports": "Sports_and_Outdoors", + "toys": "Toys_and_Games", +} + +_BASE_URL = "https://snap.stanford.edu/data/amazon/productGraph/categoryFiles" + + +def _preprocess(gz_path: str, csv_path: str) -> None: + """Parse the raw 5-core reviews into ``user,item,rating,timestamp`` rows. + + Only mechanical cleaning is applied (drop rows with a missing field); + rows are sorted chronologically per user so downstream sequential builders + receive time-ordered sessions. + """ + rows = [] + with gzip.open(gz_path, "rt", encoding="utf-8") as f: + for line in f: + r = json.loads(line) + user = r.get("reviewerID") + item = r.get("asin") + rating = r.get("overall") + timestamp = r.get("unixReviewTime") + if user is None or item is None or rating is None or timestamp is None: + continue + rows.append((user, item, float(rating), int(timestamp))) + + rows.sort(key=lambda x: (x[0], x[3])) # (user, timestamp) + + with open(csv_path, "w") as f: + for user, item, rating, timestamp in rows: + f.write(f"{user},{item},{rating},{timestamp}\n") + + +def load_feedback(category: str, version: str = "2014", fmt: str = "UIRT", reader: Reader = None) -> List: + """Load the user-item review feedback, chronologically ordered per user. + + Parameters + ---------- + category: str, required + One of ``'beauty'``, ``'sports'``, ``'toys'`` -- the three categories + used by TIGER and subsequent Semantic-ID papers. + + version: str, default: '2014' + Dataset version. Only ``'2014'`` (McAuley 5-core) is currently supported; + 2018 and 2023 are available, but 2014 is the version used throughout the Semantic-ID literature. + + fmt: str, default: 'UIRT' + Data format; the underlying file has user, item, rating, and timestamp + columns, so ``'UIR'`` and ``'UI'`` are also valid. + + reader: `obj:cornac.data.Reader`, default: None + Reader object used to read the data. + + Returns + ------- + data: array-like + Data in the form of a list of tuples (user, item, rating, timestamp). + """ + if category not in _CATEGORY_FILES: + raise ValueError(f"category='{category}' not supported; " f"choose one of {sorted(_CATEGORY_FILES)}") + if version != "2014": + raise ValueError(f"version='{version}' not supported; only '2014' (McAuley 5-core) " "is available") + + stem = _CATEGORY_FILES[category] + gz_path = cache( + url=f"{_BASE_URL}/reviews_{stem}_5.json.gz", + relative_path=f"amazon_review/{category}_{version}.json.gz", + ) + csv_path = f"{gz_path[:-len('.json.gz')]}.csv" + if not os.path.exists(csv_path): + _preprocess(gz_path, csv_path) + + reader = Reader() if reader is None else reader + return reader.read(csv_path, fmt=fmt, sep=",") diff --git a/tests/cornac/datasets/test_amazon_review.py b/tests/cornac/datasets/test_amazon_review.py new file mode 100644 index 000000000..0166ed3da --- /dev/null +++ b/tests/cornac/datasets/test_amazon_review.py @@ -0,0 +1,65 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import gzip +import json +import os +import random +import tempfile +import time +import unittest + +from cornac.datasets import amazon_review + + +class TestAmazonReview(unittest.TestCase): + + def test_preprocess_orders_chronologically_per_user(self): + # Two users with out-of-order review times; one row missing a field. + raw = [ + {"reviewerID": "u1", "asin": "iB", "overall": 4.0, "unixReviewTime": 200}, + {"reviewerID": "u1", "asin": "iA", "overall": 5.0, "unixReviewTime": 100}, + {"reviewerID": "u2", "asin": "iC", "overall": 3.0, "unixReviewTime": 150}, + {"reviewerID": "u1", "asin": "iD", "overall": 2.0}, # no timestamp -> dropped + ] + with tempfile.TemporaryDirectory() as d: + gz_path = os.path.join(d, "reviews.json.gz") + csv_path = os.path.join(d, "reviews.csv") + with gzip.open(gz_path, "wt", encoding="utf-8") as f: + for r in raw: + f.write(json.dumps(r) + "\n") + + amazon_review._preprocess(gz_path, csv_path) + + with open(csv_path) as f: + lines = [ln.strip() for ln in f if ln.strip()] + + # Missing-timestamp row dropped; u1 sorted by time (iA before iB). + self.assertEqual( + lines, + ["u1,iA,5.0,100", "u1,iB,4.0,200", "u2,iC,3.0,150"], + ) + + def test_load_feedback(self): + # only run data download tests 20% of the time to speed up frequent testing + random.seed(time.time()) + if random.random() > 0.8: + data = amazon_review.load_feedback(category="beauty") + self.assertGreater(len(data), 0) + self.assertEqual(len(data[0]), 4) # (user, item, rating, timestamp) + + +if __name__ == "__main__": + unittest.main() From c06fbb42c42e47ebc35f8a518797c07d63b418bf Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 9 Jul 2026 11:12:33 +0800 Subject: [PATCH 3/4] expand test coverage for NextItemEvaluation --- .../eval_methods/test_next_item_evaluation.py | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/cornac/eval_methods/test_next_item_evaluation.py b/tests/cornac/eval_methods/test_next_item_evaluation.py index b0c85ab91..6e3847068 100644 --- a/tests/cornac/eval_methods/test_next_item_evaluation.py +++ b/tests/cornac/eval_methods/test_next_item_evaluation.py @@ -16,11 +16,10 @@ import unittest import warnings -from cornac.eval_methods import NextItemEvaluation from cornac.data import Reader -from cornac.models import SPop +from cornac.eval_methods import NextItemEvaluation from cornac.metrics import HitRatio, Recall - +from cornac.models import SPop def _split_sids(dataset): """Raw session ids actually present in a built split.""" @@ -132,6 +131,40 @@ def test_empty_test_raises(self): self.usit, test_timestamp=1000, val_timestamp=None, fmt="USIT", ) + def test_empty_train_raises(self): + # test_timestamp below every session's last event -> nothing left for + # train (all sessions land in test). + with self.assertRaises(ValueError): + NextItemEvaluation.from_timestamps( + self.usit, + test_timestamp=5, + val_timestamp=None, + fmt="USIT", + ) + + def test_empty_val_warns(self): + # No session ends in [55, 65): train={s1,s2,s3}, test={s4,s5,s6}, val + # empty -> warn and fall back to no validation set. + with self.assertWarns(UserWarning): + m = NextItemEvaluation.from_timestamps( + self.usit, + test_timestamp=65, + val_timestamp=55, + fmt="USIT", + ) + self.assertIsNone(m.val_set) + + def test_verbose(self): + # Exercise the verbose split-summary branch. + m = NextItemEvaluation.from_timestamps( + self.usit, + test_timestamp=100, + val_timestamp=50, + fmt="USIT", + verbose=True, + ) + self.assertEqual(m.test_set.num_sessions, 2) + def test_sit_format(self): # SIT: (session, item, timestamp) -- no user column. sit = [ @@ -233,6 +266,11 @@ def test_mode_passthrough(self): m = NextItemEvaluation.leave_last_out(self.uirt, mode="next") self.assertEqual(m.mode, "next") + def test_verbose(self): + # Exercise the verbose split-summary branch. + m = NextItemEvaluation.leave_last_out(self.uirt, verbose=True) + self.assertEqual(_split_sids(m.test_set), {"u1", "u2"}) + if __name__ == "__main__": unittest.main() From cb27c9fa4827fbe2db118fc4ac1949e981fee686 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Thu, 9 Jul 2026 11:24:21 +0800 Subject: [PATCH 4/4] docs: add Amazon dataset documentation --- cornac/datasets/README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cornac/datasets/README.md b/cornac/datasets/README.md index 3dd614df2..9d1787b3d 100644 --- a/cornac/datasets/README.md +++ b/cornac/datasets/README.md @@ -281,3 +281,25 @@ Session-aware recommendation extends next-item (session-based) recommendation by | [Cosmetics](./cosmetics.py) | 17,268 | 42,367 | 172,242 | 2,533,262 | 9.97 | 59.79 | 14.71 | 0.346% | For session-based (next-item) evaluation, [Diginetica](./diginetica.py)'s `load_val()` and `load_test()` default to `mode="session-based"`, returning each user's single held-out session (`val_sbr`/`test_sbr`) with no training transitions repeated — the clean evaluation set used by session-based models such as [FPMC](../models/fpmc/) and [GRU4Rec](../models/gru4rec/). Pass `mode="session-aware"` to load the cumulative files (`val`/`test`) instead, where each user's prior sessions precede their held-out one for cross-session models. + +--- + +## Semantic-ID Datasets +### Amazon Product Review +[Amazon Product Review](./amazon_review.py) 5-core + +Each user's reviews form one chronologically-ordered sequence. Interactions are loaded via `amazon_review.load_feedback(category=...)` in `UIRT` format (user, item, rating, timestamp). No preprocessing is needed and the data is kept as-is for comparability with published results (with `leave-last-out` split). + +| Dataset | #Users | #Items | #Interactions | Type | +| :----------------------- | -----: | -----: | ------------: | :-------- | +| Amazon Beauty (`beauty`) | 22,363 | 12,101 | 198,502 | INT [1,5] | +| Amazon Sports (`sports`) | 35,598 | 18,357 | 296,337 | INT [1,5] | +| Amazon Toys (`toys`) | 19,412 | 11,924 | 167,597 | INT [1,5] | + +```Python +from cornac.datasets import amazon_review +from cornac.eval_methods import NextItemEvaluation + +data = amazon_review.load_feedback(category="beauty") # UIRT tuples, chronological per user +eval_method = NextItemEvaluation.leave_last_out(data, fmt="UIRT") +```