From aecc6700e2b2905af3f31ed5ea2dd6389870efb3 Mon Sep 17 00:00:00 2001
From: YLTsai0609 <yltsai0609@gmail.com>
Date: Tue, 16 Feb 2021 12:18:01 +0800
Subject: [PATCH 1/5] [Fix] fix test_coverage Train.shape[0] and _mkl
 libmkl_rt.dylib

---
 RecModel/base_model.py |  5 ++++-
 RecModel/utils.py      | 32 +++++++++++++++++++++++---------
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/RecModel/base_model.py b/RecModel/base_model.py
index e143325..6883010 100644
--- a/RecModel/base_model.py
+++ b/RecModel/base_model.py
@@ -190,7 +190,10 @@ def _mkl(cls):
             try:
                 cls._mkl_rt = ctypes.CDLL('libmkl_rt.so')
             except OSError:
-                cls._mkl_rt = ctypes.CDLL('mkl_rt.dll')
+                try:
+                    cls._mkl_rt = ctypes.CDLL('mkl_rt.dll')
+                except OSError:
+                    cls._mkl_rt = ctypes.CDLL('libmkl_rt.dylib')  # for someone might use miniconda
         return cls._mkl_rt
 
     @classmethod
diff --git a/RecModel/utils.py b/RecModel/utils.py
index cff54c2..04d9a11 100644
--- a/RecModel/utils.py
+++ b/RecModel/utils.py
@@ -1,21 +1,35 @@
 import numpy as np
 
+
 def test_coverage(cls, Train, topN):
-    """Testing the coverage of the algorithm:
-        It is assumed cls is a object of classes derived from RecModel and is able to rank items with a rank function.
     """
-    item_counts = np.zeros(Train.shape[0], dtype=np.int32)
+        Testing the coverage of the algorithm:
+        It is assumed cls is a object of classes derived 
+        from RecModel and is able to rank items with 
+        a rank function.
+
+        `idxptr` : row index of sparse matrix
+        https://stackoverflow.com/questions/52299420/scipy-csr-matrix-understand-indptr
+
+        :param cls : RecModel
+        :param Train : scipy.sparse.csr_matrix, shape (Users, Items)
+        :param topN : int, top N items to be select by recommender
+    """
+    item_counts = np.zeros(Train.shape[1], dtype=np.int32)
 
     for user in range(Train.shape[0]):
         start_usr = Train.indptr[user]
-        end_usr = Train.indptr[user+1]
+        end_usr = Train.indptr[user + 1]
 
-        items_to_rank = np.delete(np.arange(Train.shape[1], dtype=np.int32), Train.indices[start_usr:end_usr])
-        ranked_items = cls.rank(users=user, items=items_to_rank, topn=topN).reshape(-1)
+        items_to_rank = np.delete(
+            np.arange(Train.shape[1], dtype=np.int32), Train.indices[start_usr:end_usr])
+        ranked_items = cls.rank(
+            users=user, items=items_to_rank, topn=topN).reshape(-1)
         item_counts[ranked_items[:topN]] += 1
-    
+
     return item_counts
 
+
 def train_test_split_sparse_mat(matrix, train=0.8, seed=1993):
     np.random.seed(seed)
     train_data = matrix.tocoo()
@@ -31,5 +45,5 @@ def train_test_split_sparse_mat(matrix, train=0.8, seed=1993):
 
     train_data.eliminate_zeros()
     test_data.eliminate_zeros()
-    
-    return [train_data, test_data]
\ No newline at end of file
+
+    return [train_data, test_data]

From bc84b4214beaf483cfe6db5230bbba135052ee1b Mon Sep 17 00:00:00 2001
From: YLTsai0609 <yltsai0609@gmail.com>
Date: Thu, 18 Feb 2021 01:41:40 +0800
Subject: [PATCH 2/5] [Fix, Config] fix eval_topn random selected item without
 replacement and func doc

---
 RecModel/base_model.py | 58 +++++++++++++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/RecModel/base_model.py b/RecModel/base_model.py
index 6883010..d25e9e4 100644
--- a/RecModel/base_model.py
+++ b/RecModel/base_model.py
@@ -7,6 +7,7 @@
 import ctypes
 import sharedmem
 
+
 def iter_rows_two_matrices(A, B):
     """
     Idea from FROM https://github.com/benanne/wmf/blob/master/wmf.py
@@ -46,6 +47,17 @@ def predict(self, user_item):
         pass
 
     def rank(self, items, user, topn=None):
+        """
+        generate recommendation by model from given items and user
+        Args:
+            items (np.ndarray - shape(N,)): the items random sampled from all items, size N
+            user ([int, np.ndarray - shape(N, )]): single user or multiple user in a np.ndarray, size N
+            topn ([int], optional): top n items to retrun as recommendation. Defaults to None.
+
+        Return:
+
+        """
+
         pass
 
     def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
@@ -59,9 +71,10 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
 
             # Select a sample of rand_sampled items the user never bought.
 
-            # Sample some random items
-            rand_items = np.random.randint(0, self.num_items, size=(rand_sampled + 1))
-            rand_pos = np.random.randint(0, rand_sampled- (2 * topn.max()))
+            # Sample some random items (the rand_items are unique)
+            rand_items = np.random.choice(
+                self.num_items, size=(rand_sampled + 1), replace=False)
+            rand_pos = np.random.randint(0, rand_sampled - (2 * topn.max()))
 
             # Did the user buy any of them?
             # This is excluded for performance reasons.
@@ -87,7 +100,8 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
 
                 # Get the topn items form the random sampled items plus the truly bought item
                 rand_items[rand_pos] = item
-                candidates = self.rank(items=rand_items, users=user, topn=topn.max())
+                candidates = self.rank(
+                    items=rand_items, users=user, topn=topn.max())
 
                 #candidates = self.rank(items=np.insert(arr=rand_items, obj=int(rand_sampled * 0.5), values=item), users=user, topn=topn.max())
 
@@ -97,14 +111,17 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
                         sum_hits[pos] += 1
             return sum_hits
 
-    def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sampled =1000, cores=1, random_state=None, dtype='float32'):
+    def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sampled=1000,
+                  cores=1, random_state=None, dtype='float32'):
         """
         Ranking evaluation of models (for topn prediction).
-        :param test_mat: The actual matrix that should be evaluated on.
-        :param train_mat: To evaluate, for each user, random items that the user did not buy will be sampled. Therefore,
-        all other matrixes have to be known to make sure that the random items were not bought by that user.
-        :param eval_mat: To evaluate, for each user, random items that the user did not buy will be sampled. Therefore,
-        all other matrixes have to be known to make sure that the random items were not bought by that user.
+        :param test_mat: (sparse matrix) shape : (Users, Items) - The actual matrix that should be evaluated on.
+        :param train_mat: (sparse matrix) shape : (Users, Items) - To evaluate, for each user, random items that the user 
+            did not buy will be sampled. Therefore,
+            all other matrixes have to be known to make sure that the random items were not bought by that user.
+        :param eval_mat: (sparse matrix) shape : (Users, Items) - To evaluate, for each user, random items that the user 
+            did not buy will be sampled. Therefore,
+            all other matrixes have to be known to make sure that the random items were not bought by that user.
         :param topn: How many items should be looked for to find the potential hits?
         :param metric: Which metric should be used for evaluation? Should be one of  ARHR, PRECISION, RECALL
         :return: Evaluation score.
@@ -126,18 +143,20 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sam
         if cores == 1:
             with MKLThreads(1):
                 for elem in iter_rows_two_matrices(super_mat, test_mat):
-                    hits += self.compute_hit(elem, rand_sampled=rand_sampled, topn=topn)
+                    hits += self.compute_hit(elem,
+                                             rand_sampled=rand_sampled, topn=topn)
         else:
             with MKLThreads(1):
                 # Parallel computation of the recall.
                 pool = Pool(cores)
-                compute_hit_args = partial(self.compute_hit, rand_sampled=rand_sampled, topn=topn)
+                compute_hit_args = partial(
+                    self.compute_hit, rand_sampled=rand_sampled, topn=topn)
                 hits = np.stack(
                     (pool.map(compute_hit_args,
-                                (elem for elem in iter_rows_two_matrices(super_mat, test_mat))))).sum(axis=0)
+                              (elem for elem in iter_rows_two_matrices(super_mat, test_mat))))).sum(axis=0)
                 pool.close()
                 pool.join()
-                
+
         # Compute the precision at topn
         recall_dict = {}
         recall = hits / len(test_mat.nonzero()[0])
@@ -146,7 +165,7 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sam
             recall_dict[f"Recall@{topn[pos]}"] = recall[pos]
 
         return recall_dict
-        
+
     def eval_prec(self, utility_mat, metric='mse'):
         """
         Accuaracy-based evaluation of models.
@@ -163,7 +182,8 @@ def eval_prec(self, utility_mat, metric='mse'):
             non_zero_elements = utility_mat.nonzero()
 
             # Get predictions from the model
-            predictions = self.predict(users=non_zero_elements[0], items=non_zero_elements[1]).reshape(1, -1)
+            predictions = self.predict(
+                users=non_zero_elements[0], items=non_zero_elements[1]).reshape(1, -1)
 
             # Apply the corresponding eval metric to get result.
             if metric == 'RMSE':
@@ -178,6 +198,7 @@ def eval_prec(self, utility_mat, metric='mse'):
         else:
             raise ValueError("Metric {metric} is not implemented.")
 
+
 class MKLThreads(object):
     """
     Multithreading with MKL from https://stackoverflow.com/questions/28283112/using-mkl-set-num-threads-with-numpy
@@ -193,7 +214,8 @@ def _mkl(cls):
                 try:
                     cls._mkl_rt = ctypes.CDLL('mkl_rt.dll')
                 except OSError:
-                    cls._mkl_rt = ctypes.CDLL('libmkl_rt.dylib')  # for someone might use miniconda
+                    # for someone might use miniconda
+                    cls._mkl_rt = ctypes.CDLL('libmkl_rt.dylib')
         return cls._mkl_rt
 
     @classmethod
@@ -214,4 +236,4 @@ def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
-        self.set_num_threads(self._saved_n)
\ No newline at end of file
+        self.set_num_threads(self._saved_n)

From 9d264d01a996fdc9d8928a7458c9bc9f264f980e Mon Sep 17 00:00:00 2001
From: YLTsai0609 <yltsai0609@gmail.com>
Date: Fri, 19 Feb 2021 02:00:59 +0800
Subject: [PATCH 3/5] [Config] rename variavle ib base_model.py compute_hit
 func line 69 for better understanding

---
 RecModel/base_model.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/RecModel/base_model.py b/RecModel/base_model.py
index d25e9e4..95d4d3b 100644
--- a/RecModel/base_model.py
+++ b/RecModel/base_model.py
@@ -12,6 +12,11 @@ def iter_rows_two_matrices(A, B):
     """
     Idea from FROM https://github.com/benanne/wmf/blob/master/wmf.py
     Getting the indices of each user for two matrices. Matrice A, B need the same number of rows!
+
+    Check the https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html
+    for csr_matrix meaning, 
+    csr_matrix.data means the nonzero elements in np.array format
+    csr_matrix.indices means the row index of nonzero elements in np.array format
     """
     # Make this save for empty rows!
 
@@ -61,13 +66,13 @@ def rank(self, items, user, topn=None):
         pass
 
     def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
-        user, super_mat_dat, super_mat_idx, test_mat_dat, test_mat_idx = elem
-        if len(test_mat_dat) == 0:
+        user, super_mat_user_items, super_mat_user_items_idx, test_mat_user_items, test_mat_user_items_idx = elem
+        if len(test_mat_user_items) == 0:
             # Not a single item picked for this user in the test, mat
             return np.zeros(topn.shape, dtype=dtype)
         else:
             # Get all items the user bought in any of the matrices (i.e. the supermat)
-            items_selected = super_mat_idx
+            items_selected = super_mat_user_items_idx
 
             # Select a sample of rand_sampled items the user never bought.
 
@@ -96,7 +101,7 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
                 rand_items = np.append(rand_items, new_sample)"""
 
             sum_hits = np.zeros(topn.shape, dtype=dtype)
-            for item in test_mat_idx:
+            for item in test_mat_user_items_idx:
 
                 # Get the topn items form the random sampled items plus the truly bought item
                 rand_items[rand_pos] = item
@@ -125,7 +130,14 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sam
         :param topn: How many items should be looked for to find the potential hits?
         :param metric: Which metric should be used for evaluation? Should be one of  ARHR, PRECISION, RECALL
         :return: Evaluation score.
+
+        TODO
+            add param : n_users_in_test(int) default = None
+            modify param : rand_sampled(int) -> n_rand_sampled_items(int)
         """
+        # TODO
+        # assert n_users in test_mat
+        # sampling user in test_mat
         super_mat = test_mat
         if not train_mat is None:
             super_mat += train_mat
@@ -137,7 +149,8 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sam
         # if topn is not list make is one.
         if not isinstance(topn, np.ndarray):
             raise ValueError("Topn has to be a np.array")
-        # For each user, for each item select he interacted with, sample topn not selected items. Then let the model
+        # For each user, for each item select he interacted with,
+        # sample topn not selected items. Then let the model
         # rank the topn + 1 items at compare were the acutal by was ranked.
         hits = np.zeros(topn.shape, dtype=dtype)
         if cores == 1:

From 185ed56fe065fd01e313a5997af432aba5ad1f18 Mon Sep 17 00:00:00 2001
From: YLTsai0609 <yltsai0609@gmail.com>
Date: Tue, 23 Feb 2021 00:21:45 +0800
Subject: [PATCH 4/5] [Feature] add random sampling user for eval_topn

---
 RecModel/base_model.py | 83 +++++++++++++++++++++++++-----------------
 1 file changed, 50 insertions(+), 33 deletions(-)

diff --git a/RecModel/base_model.py b/RecModel/base_model.py
index 95d4d3b..9a54bca 100644
--- a/RecModel/base_model.py
+++ b/RecModel/base_model.py
@@ -14,7 +14,7 @@ def iter_rows_two_matrices(A, B):
     Getting the indices of each user for two matrices. Matrice A, B need the same number of rows!
 
     Check the https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html
-    for csr_matrix meaning, 
+    for csr_matrix meaning,
     csr_matrix.data means the nonzero elements in np.array format
     csr_matrix.indices means the row index of nonzero elements in np.array format
     """
@@ -58,14 +58,13 @@ def rank(self, items, user, topn=None):
             items (np.ndarray - shape(N,)): the items random sampled from all items, size N
             user ([int, np.ndarray - shape(N, )]): single user or multiple user in a np.ndarray, size N
             topn ([int], optional): top n items to retrun as recommendation. Defaults to None.
-
         Return:
-
         """
 
         pass
 
     def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
+        # TODO rename the 5 element in the tuple for better understanding
         user, super_mat_user_items, super_mat_user_items_idx, test_mat_user_items, test_mat_user_items_idx = elem
         if len(test_mat_user_items) == 0:
             # Not a single item picked for this user in the test, mat
@@ -93,11 +92,13 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
 
                 # How many samples do we have to draw again?
                 missing_items = rand_sampled - len(rand_items)
-                new_sample = np.random.randint(0, self.num_items, size=missing_items)
+                new_sample = np.random.randint(
+                    0, self.num_items, size=missing_items)
 
                 # If the newly sampled items still contain items that the user did consume, redo the sample.
                 while np.isin(new_sample, items_selected).any():
-                    new_sample = np.random.randint(0, self.num_items, size=missing_items)
+                    new_sample = np.random.randint(
+                        0, self.num_items, size=missing_items)
                 rand_items = np.append(rand_items, new_sample)"""
 
             sum_hits = np.zeros(topn.shape, dtype=dtype)
@@ -108,7 +109,7 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
                 candidates = self.rank(
                     items=rand_items, users=user, topn=topn.max())
 
-                #candidates = self.rank(items=np.insert(arr=rand_items, obj=int(rand_sampled * 0.5), values=item), users=user, topn=topn.max())
+                # candidates = self.rank(items=np.insert(arr=rand_items, obj=int(rand_sampled * 0.5), values=item), users=user, topn=topn.max())
 
                 # If the true item was in the topn we have a hit!
                 for pos in range(len(topn)):
@@ -116,39 +117,55 @@ def compute_hit(self, elem, rand_sampled, topn, dtype="float32"):
                         sum_hits[pos] += 1
             return sum_hits
 
-    def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sampled=1000,
+    def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10],
+                  rand_sampled_users=1000,
+                  rand_sampled_items=1000,
                   cores=1, random_state=None, dtype='float32'):
+        """Ranking evaluation of models (for topn prediction)
+
+        Args:
+            test_mat (csr_matrix) : shape : (Users, Items) The actual matrix that should be evaluated on.
+            train_mat (csr_matrix, optional): shape : (Users, Items) - To evaluate, for each user, random items that the user 
+                did not buy will be sampled. Therefore,
+                all other matrixes have to be known to make sure that the random items were not bought by that user. Default is None
+            eval_mat (csr_matrix, optional): shape : (Users, Items) - To evaluate, for each user, random items that the user. Defaults to None.
+            topn (list, optional): How many items should be looked for to find the potential hits?. Defaults to [10].
+            rand_sampled_users (int, optional): How many users will be sampled in test_mat, sampling for efficient evaluation?. Defaults to 1000.
+            rand_sampled_items (int, optional): How many items will be sampled when generate recommendation?. Defaults to 1000.
+            cores (int, optional): how many cores you wanna use. Defaults to 1.
+            random_state (int, optional): The random state for rand_sampled_users and rand_sampled_items. Defaults to None.
+            dtype (str, optional): dtype when we create hit matrix Defaults to 'float32'.
+
+        Raises:
+            ValueError: argument topn should be a np.ndarray
+
+        Returns:
+            dict: return recall@N in dict format
         """
-        Ranking evaluation of models (for topn prediction).
-        :param test_mat: (sparse matrix) shape : (Users, Items) - The actual matrix that should be evaluated on.
-        :param train_mat: (sparse matrix) shape : (Users, Items) - To evaluate, for each user, random items that the user 
-            did not buy will be sampled. Therefore,
-            all other matrixes have to be known to make sure that the random items were not bought by that user.
-        :param eval_mat: (sparse matrix) shape : (Users, Items) - To evaluate, for each user, random items that the user 
-            did not buy will be sampled. Therefore,
-            all other matrixes have to be known to make sure that the random items were not bought by that user.
-        :param topn: How many items should be looked for to find the potential hits?
-        :param metric: Which metric should be used for evaluation? Should be one of  ARHR, PRECISION, RECALL
-        :return: Evaluation score.
-
-        TODO
-            add param : n_users_in_test(int) default = None
-            modify param : rand_sampled(int) -> n_rand_sampled_items(int)
-        """
-        # TODO
-        # assert n_users in test_mat
-        # sampling user in test_mat
+        assert rand_sampled_users is None or rand_sampled_users > 0, f'The number of test users ({rand_sampled_users}) should be > 0.'
+        assert rand_sampled_items > 0, f'The number of random sampling items ({rand_sampled_items}) should be > 0.'
+        # if topn is not list make is one.
+        if not isinstance(topn, np.ndarray):
+            raise ValueError("Topn has to be a np.ndarray")
+        if rand_sampled_users is None:
+            rand_sampled_users = test_mat.shape[0]
+        else:
+            rand_sampled_users = test_mat.shape[0] if rand_sampled_users is None else min(
+                rand_sampled_users, test_mat.shape[0])
+        print(f'This process will sampling {rand_sampled_users}')
+
+        if not random_state is None:
+            np.random.seed(random_state)
+        rand_user_ids = np.random.choice(
+            np.arange(test_mat.shape[0]), size=rand_sampled_users, replace=False)
+        test_mat = test_mat[rand_user_ids, :]
+
         super_mat = test_mat
         if not train_mat is None:
             super_mat += train_mat
         if not eval_mat is None:
             super_mat += eval_mat
-        if not random_state is None:
-            np.random.seed(random_state)
 
-        # if topn is not list make is one.
-        if not isinstance(topn, np.ndarray):
-            raise ValueError("Topn has to be a np.array")
         # For each user, for each item select he interacted with,
         # sample topn not selected items. Then let the model
         # rank the topn + 1 items at compare were the acutal by was ranked.
@@ -157,13 +174,13 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10], rand_sam
             with MKLThreads(1):
                 for elem in iter_rows_two_matrices(super_mat, test_mat):
                     hits += self.compute_hit(elem,
-                                             rand_sampled=rand_sampled, topn=topn)
+                                             rand_sampled=rand_sampled_items, topn=topn)
         else:
             with MKLThreads(1):
                 # Parallel computation of the recall.
                 pool = Pool(cores)
                 compute_hit_args = partial(
-                    self.compute_hit, rand_sampled=rand_sampled, topn=topn)
+                    self.compute_hit, rand_sampled=rand_sampled_items, topn=topn)
                 hits = np.stack(
                     (pool.map(compute_hit_args,
                               (elem for elem in iter_rows_two_matrices(super_mat, test_mat))))).sum(axis=0)

From ff186be935ce238f3738ec780914eb39dcfe7350 Mon Sep 17 00:00:00 2001
From: YLTsai0609 <yltsai0609@gmail.com>
Date: Wed, 17 Mar 2021 01:25:52 +0800
Subject: [PATCH 5/5] [Feature] add sampling on coverage

---
 RecModel/base_model.py    |  3 ++
 RecModel/recwalk_model.py | 76 +++++++++++++++++++++++++--------------
 RecModel/slim_model.py    | 30 +++++++++-------
 RecModel/utils.py         | 16 +++++++--
 requirements.txt          |  0
 setup.py                  |  6 ++++
 6 files changed, 90 insertions(+), 41 deletions(-)
 create mode 100644 requirements.txt
 create mode 100644 setup.py

diff --git a/RecModel/base_model.py b/RecModel/base_model.py
index 9a54bca..8003bb3 100644
--- a/RecModel/base_model.py
+++ b/RecModel/base_model.py
@@ -141,6 +141,9 @@ def eval_topn(self, test_mat, train_mat=None, eval_mat=None, topn=[10],
 
         Returns:
             dict: return recall@N in dict format
+        TODO : efficiency improvement, need a efficient way to acess the sparse matrix
+        SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.
+        self._set_arrayXarray(i, j, x)
         """
         assert rand_sampled_users is None or rand_sampled_users > 0, f'The number of test users ({rand_sampled_users}) should be > 0.'
         assert rand_sampled_items > 0, f'The number of random sampling items ({rand_sampled_items}) should be > 0.'
diff --git a/RecModel/recwalk_model.py b/RecModel/recwalk_model.py
index d3cc3b7..8a845fd 100644
--- a/RecModel/recwalk_model.py
+++ b/RecModel/recwalk_model.py
@@ -4,13 +4,15 @@
 import ctypes
 import os
 
-# Imports from own package. 
+# Imports from own package.
 from RecModel.base_model import RecModel
 from RecModel.fast_utils.slim_utils import _predict_slim, train_Slim
 
 # Helper functions
+
+
 def fill_empty_row_or_col(mat, fill_value=1.0):
-    mat=mat.copy()
+    mat = mat.copy()
 
     # First fill the empty rows
     empty_rows = (mat.sum(axis=1).A1 == 0)
@@ -22,24 +24,28 @@ def fill_empty_row_or_col(mat, fill_value=1.0):
     # If there are some columns remaining zero also fill them!
     empty_cols = (mat.sum(axis=0).A1 == 0)
     if empty_cols.any():
-        random_users=np.random.randint(0, mat.shape[0], empty_cols.sum())
+        random_users = np.random.randint(0, mat.shape[0], empty_cols.sum())
         mat[random_users, empty_cols] = fill_value
     return mat
 
+
 def create_A_g(mat):
     n_users, n_items = mat.shape
 
     # Built upper half.
-    zeros_upper_left = scipy.sparse.csr_matrix((n_users, n_users), dtype=np.float32) 
+    zeros_upper_left = scipy.sparse.csr_matrix(
+        (n_users, n_users), dtype=np.float32)
     upper_half = scipy.sparse.hstack([zeros_upper_left, mat], format='csr')
 
     # Build lower half.
-    zeros_lower_right = scipy.sparse.csr_matrix((n_items, n_items), dtype=np.float32)    
+    zeros_lower_right = scipy.sparse.csr_matrix(
+        (n_items, n_items), dtype=np.float32)
     lower_half = scipy.sparse.hstack([mat.T, zeros_lower_right], format='csr')
 
     A_g = scipy.sparse.vstack([upper_half, lower_half], format='csr')
     return A_g
 
+
 def create_H(mat):
     A_g = create_A_g(mat)
 
@@ -47,15 +53,18 @@ def create_H(mat):
     row_sums = A_g.sum(axis=1).A1
 
     # fill the digonal matrix with the inverse of row sums.
-    diag_inv_row_sum = scipy.sparse.diags(diagonals=(1 / row_sums), offsets=0, dtype=np.float32, format='csr')
+    diag_inv_row_sum = scipy.sparse.diags(diagonals=(
+        1 / row_sums), offsets=0, dtype=np.float32, format='csr')
 
     return diag_inv_row_sum.dot(A_g)
 
+
 def create_M_i(W_indptr, W_indices, W_data, n_items):
     """
     M_i is create by making W row stochastic. 
     """
-    W = scipy.sparse.csr_matrix((W_data, W_indices, W_indptr), shape=(n_items, n_items), dtype=np.float32)
+    W = scipy.sparse.csr_matrix((W_data, W_indices, W_indptr), shape=(
+        n_items, n_items), dtype=np.float32)
     W_normalized = W.copy()
 
     # Compute maximal row sums
@@ -66,20 +75,25 @@ def create_M_i(W_indptr, W_indices, W_data, n_items):
     W_normalized.data /= row_sum_max
 
     # Create diagonal mat that reintroduces the residuals to make the final Matrix row stochastic.
-    diag_mat = scipy.sparse.diags(diagonals = 1 - (row_sums / row_sum_max), offsets=0, dtype=np.float32, format='csr')
+    diag_mat = scipy.sparse.diags(
+        diagonals=1 - (row_sums / row_sum_max), offsets=0, dtype=np.float32, format='csr')
 
     M_i = W_normalized + diag_mat
 
     return M_i
 
+
 def create_M(indptr, indices, data, n_users, n_items):
     # Make W row stochastic.
     M_i = create_M_i(indptr, indices, data, n_items)
 
     # Fill the rest of the matrix with zeros and the upper left with an identity matrix.
-    I = scipy.sparse.diags(diagonals=np.full(n_users, 1, dtype=np.float32), offsets=0, dtype=np.float32, format='csr')
-    zeros_upper_right = scipy.sparse.csr_matrix((n_users, n_items), dtype=np.float32)    
-    zeros_lower_left = scipy.sparse.csr_matrix((n_items, n_users), dtype=np.float32)    
+    I = scipy.sparse.diags(diagonals=np.full(
+        n_users, 1, dtype=np.float32), offsets=0, dtype=np.float32, format='csr')
+    zeros_upper_right = scipy.sparse.csr_matrix(
+        (n_users, n_items), dtype=np.float32)
+    zeros_lower_left = scipy.sparse.csr_matrix(
+        (n_items, n_users), dtype=np.float32)
 
     upper_half = scipy.sparse.hstack([I, zeros_upper_right], format='csr')
     lower_half = scipy.sparse.hstack([zeros_lower_left, M_i], format='csr')
@@ -88,13 +102,16 @@ def create_M(indptr, indices, data, n_users, n_items):
     return M
 
 # Recwalk model
+
+
 class Recwalk(RecModel):
     def __init__(self, num_items, num_users, k_steps, eval_method, damping=None, slim_W=None):
-        
+
         if not eval_method in ['k_step', 'PR']:
-            raise ValueError(f"eval method needs to be one of ['k_step', 'PR], not {eval_method}")
+            raise ValueError(
+                f"eval method needs to be one of ['k_step', 'PR], not {eval_method}")
 
-        self.num_users = num_users       
+        self.num_users = num_users
         self.num_items = num_items
         self.k = k_steps
         self.eval_method = eval_method
@@ -102,7 +119,8 @@ def __init__(self, num_items, num_users, k_steps, eval_method, damping=None, sli
 
         if eval_method == 'PR':
             if damping is None:
-                raise ValueError("If you want to use the power method to rank the items (eval_method = 'PR') please provide the damping factor.")
+                raise ValueError(
+                    "If you want to use the power method to rank the items (eval_method = 'PR') please provide the damping factor.")
             self.damping = damping
 
         if not slim_W is None:
@@ -111,10 +129,10 @@ def __init__(self, num_items, num_users, k_steps, eval_method, damping=None, sli
         else:
             self.slim_trained = False
 
-    def rank(self, items, users, topn):     
+    def rank(self, items, users, topn):
         # Create the representation of the user:
         user_vec = np.zeros(self.num_items + self.num_users, dtype=np.float32)
-        user_vec[users] = 1.0  
+        user_vec[users] = 1.0
 
         # Choose the prediction method.
         if self.eval_method == 'k_step':
@@ -123,23 +141,26 @@ def rank(self, items, users, topn):
             for _ in range(self.k):
                 user_vec = scipy.sparse.csr_matrix.dot(self.P, user_vec)
             predictions = user_vec[items + self.num_users]
-            
+
         else:
             # Prediction based on the stationary distribution with restarts.
             # make hard copy of user_vec as user_vec is used throughout the computation.
             vec_out = user_vec.copy()
             for _ in range(self.k):
-                vec_out = self.damping * scipy.sparse.csr_matrix.dot(self.P, vec_out) + (1-self.damping) * user_vec
+                vec_out = self.damping * \
+                    scipy.sparse.csr_matrix.dot(
+                        self.P, vec_out) + (1 - self.damping) * user_vec
                 vec_out = vec_out / (np.linalg.norm(vec_out) + 1e-10)
             predictions = vec_out[items + self.num_users]
 
         # Extract the relevant predictions from the computed vector (take the relevant item nodes)
-        
+
         # Finally sort the prediction and return the relevant items.
         return items[np.argpartition(predictions, list(range(-topn, 0, 1)))[-topn:]][::-1]
 
     def predict(self, users, items):
-        raise NotImplementedError("Predict is not defined for the RecWalk method.") 
+        raise NotImplementedError(
+            "Predict is not defined for the RecWalk method.")
 
     def train(self, train_mat, phi, alpha, l1_ratio, max_iter, tolerance, cores, verbose):
         """
@@ -161,12 +182,14 @@ def train(self, train_mat, phi, alpha, l1_ratio, max_iter, tolerance, cores, ver
 
         # Train underlying slim model.
         if self.slim_trained is False:
-            # Train slim 
-            indptr, indices, data = train_Slim(X=train_mat_csc, alpha=alpha, l1_ratio=l1_ratio, max_iter=max_iter, tol=tolerance, cores=cores, verbose=verbose)
+            # Train slim
+            indptr, indices, data = train_Slim(
+                X=train_mat_csc, alpha=alpha, l1_ratio=l1_ratio, max_iter=max_iter, tol=tolerance, cores=cores, verbose=verbose)
         else:
             # Use pre-trained slim weights
-            indptr, indices, data = (self.slim_W.indptr, self.slim_W.indices, self.slim_W.data)
-        
+            indptr, indices, data = (
+                self.slim_W.indptr, self.slim_W.indices, self.slim_W.data)
+
         # Fill the empty rows / columsn of the train mat with an entry of 1.0 at a random position to ensure stochasticity of the matrix.
         train_mat = fill_empty_row_or_col(train_mat)
 
@@ -191,7 +214,8 @@ def save_P(self, dir='P.npy'):
         if self.P is not None:
             np.save(dir, self.P)
         else:
-            raise ValueError("There is no computed P matrix because the model was not trained yet. Please train it with Recwalker.train().")
+            raise ValueError(
+                "There is no computed P matrix because the model was not trained yet. Please train it with Recwalker.train().")
 
     def load_P(self, dir='P.npy'):
         try:
diff --git a/RecModel/slim_model.py b/RecModel/slim_model.py
index f8388e5..a03c647 100644
--- a/RecModel/slim_model.py
+++ b/RecModel/slim_model.py
@@ -5,17 +5,20 @@
 import ctypes
 import os
 
-# Imports from own package. 
+# Imports from own package.
 from RecModel.base_model import RecModel
 from RecModel.fast_utils.slim_utils import _predict_slim, train_Slim
 
+
 class Slim(RecModel):
     def __init__(self, num_items, num_users):
         self.num_users = num_users
         self.num_items = num_items
 
-    def rank(self, items, users, topn):           
-        predictions = self.predict(np.full(len(items), users, dtype=np.int32), items.astype(np.int32))
+    def rank(self, items, users, topn):
+        predictions = self.predict(
+            np.full(len(items), users, dtype=np.int32),
+            items.astype(np.int32))
         return items[np.argpartition(predictions, list(range(-topn, 0, 1)))[-topn:]][::-1]
 
     def train(self, X, alpha, l1_ratio, max_iter, tolerance, cores, verbose):
@@ -26,10 +29,11 @@ def train(self, X, alpha, l1_ratio, max_iter, tolerance, cores, verbose):
 
         X_train = X.tocsc()
         X_train.sort_indices()
-        
-        self.W_indptr, self.W_idx, self.W_data = train_Slim(X=X_train, alpha=alpha, l1_ratio=l1_ratio, max_iter=max_iter, tol=tolerance, cores=cores, verbose=verbose)
-        
-        # To predit a csr matrix is needed.        
+
+        self.W_indptr, self.W_idx, self.W_data = train_Slim(
+            X=X_train, alpha=alpha, l1_ratio=l1_ratio, max_iter=max_iter, tol=tolerance, cores=cores, verbose=verbose)
+
+        # To predit a csr matrix is needed.
         X_eval = X.tocsr()
         X_eval.sort_indices()
         self.A_indptr = X_eval.indptr
@@ -38,14 +42,16 @@ def train(self, X, alpha, l1_ratio, max_iter, tolerance, cores, verbose):
 
     def predict(self, users, items):
 
-        # Either items of length 1 and users of different length, 
+        # Either items of length 1 and users of different length,
         if (users.shape[0] != items.shape[0]):
-            raise ValueError(f"Users and items need to have the same shape, but users.shape[0]={users.shape[0]} and items.shape[0]={items.shape[0]} is not possible!")
+            raise ValueError(
+                f"Users and items need to have the same shape, but users.shape[0]={users.shape[0]} and items.shape[0]={items.shape[0]} is not possible!")
 
         # Allocate output array
         pred = np.empty(items.shape[0], dtype=np.float64)
-        _predict_slim(users, items, pred, self.A_indptr, self.W_indptr, self.A_idx, self.W_idx, self.A_data, self.W_data)
-        
+        _predict_slim(users, items, pred, self.A_indptr, self.W_indptr,
+                      self.A_idx, self.W_idx, self.A_data, self.W_data)
+
         return pred
 
     @property
@@ -54,4 +60,4 @@ def W(self):
         idx = np.array(self.W_idx).copy()
         data = np.array(self.W_data).copy()
 
-        return scipy.sparse.csc_matrix((data, idx, indptr), dtype=np.float64, shape = (self.num_items, self.num_items))
\ No newline at end of file
+        return scipy.sparse.csc_matrix((data, idx, indptr), dtype=np.float64, shape=(self.num_items, self.num_items))
diff --git a/RecModel/utils.py b/RecModel/utils.py
index 04d9a11..bb6e59b 100644
--- a/RecModel/utils.py
+++ b/RecModel/utils.py
@@ -1,7 +1,7 @@
 import numpy as np
 
 
-def test_coverage(cls, Train, topN):
+def test_coverage(cls, Train, topN, rand_sampled_users=1000, random_state=None):
     """
         Testing the coverage of the algorithm:
         It is assumed cls is a object of classes derived 
@@ -14,10 +14,20 @@ def test_coverage(cls, Train, topN):
         :param cls : RecModel
         :param Train : scipy.sparse.csr_matrix, shape (Users, Items)
         :param topN : int, top N items to be select by recommender
+        :rand_sampled_users : int, random sampled user for efficient evaluation, defaults to 1000
+        :param random_state : int, seed for rand_sampled_users defaults to None
     """
+    assert rand_sampled_users is None or rand_sampled_users > 0, f'The number of test users ({rand_sampled_users}) should be > 0.'
+    if rand_sampled_users is None:
+        rand_sampled_users = Train.shape[0]
+    else:
+        rand_sampled_users = Train.shape[0] if rand_sampled_users is None else min(
+            rand_sampled_users, Train.shape[0])
+    print(f'This process will sampling {rand_sampled_users}')
+    rand_user_ids = np.random.choice(
+        np.arange(Train.shape[0]), size=rand_sampled_users, replace=False)
     item_counts = np.zeros(Train.shape[1], dtype=np.int32)
-
-    for user in range(Train.shape[0]):
+    for user in rand_user_ids:
         start_usr = Train.indptr[user]
         end_usr = Train.indptr[user + 1]
 
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..e69de29
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..695bad9
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,6 @@
+
+from setuptools import setup
+setup(
+    name='RecModel',  # Required
+    description='RecModel',  # Optional
+)