Merge pull request #12 from CoolCoderCarl/develop

Develop

Author: CoolCoderCarl

README.md

@@ -1,4 +1,4 @@
-# truth_seeker
+# datapath
 
 Simple ETL getting info from https://newsapi.org/ and send to telegram channel
 
@@ -19,7 +19,7 @@ You can check the last available tags here -
 Need to fill `settings.toml` with next important variables:
 1) `API_KEY`. You can find this data here - https://my.telegram.org/apps
 2) `API_TOKEN`. Ask *BotFather* in telegram.
-3) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/bot<API_TOKEN>/getUpdates
+3) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/botAPI_TOKEN/getUpdates
 4) `DB_NAME`. Where you want to load your data before send to telegram.
 5) `QUERY`. Key word to search for in articles.
 6) `docker-compose up -d`

datapath.py

@@ -7,22 +7,25 @@
 import dynaconfig
 import news_db
 
-# Time range for sending messages
-TIME_TO_SEND_START = datetime.now().replace(hour=10, minute=00).strftime("%H:%M")
-TIME_TO_SEND_END = datetime.now().replace(hour=20, minute=00).strftime("%H:%M")
+# Time range for sending messages loaded from settings.toml
+TIME_TO_SEND_START = dynaconfig.settings["TIMINIGS"]["TIME_TO_SEND_START"]
+TIME_TO_SEND_END = dynaconfig.settings["TIMINIGS"]["TIME_TO_SEND_END"]
 
 # Logging
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO
 )
+logging.basicConfig(
+    format="%(asctime)s - %(levelname)s - %(message)s", level=logging.WARNING
+)
 logging.basicConfig(
     format="%(asctime)s - %(levelname)s - %(message)s", level=logging.ERROR
 )
 
 
-API_TOKEN = dynaconfig.settings["API_TOKEN"]
+API_TOKEN = dynaconfig.settings["TELEGRAM"]["API_TOKEN"]
 API_URL = f"https://api.telegram.org/bot{API_TOKEN}/sendMessage"
-CHAT_ID = dynaconfig.settings["CHAT_ID"]
+CHAT_ID = dynaconfig.settings["TELEGRAM"]["CHAT_ID"]
 
 
 def send_news_to_telegram(message):
@@ -32,7 +35,21 @@ def send_news_to_telegram(message):
     :return:
     """
     try:
-        response = requests.post(API_URL, json={"chat_id": CHAT_ID, "text": message})
+        response = requests.post(
+            API_URL,
+            json={
+                "chat_id": CHAT_ID,
+                "text": f"Author: {message[0]}\n"
+                f"\n"
+                f"Title: {message[1]}\n"
+                f"\n"
+                f"{message[2]}\n"
+                f"\n"
+                f"URL: {message[3]}\n"
+                f"\n"
+                f"Date published: {message[4]}\n",
+            },
+        )
         if response.status_code == 200:
             logging.info(
                 f"Sent: {response.reason}. Status code: {response.status_code}"
@@ -46,14 +63,21 @@ def send_news_to_telegram(message):
 
 
 if __name__ == "__main__":
-    data_from_db = news_db.create_connection(news_db.DB_FILE)
+    db_connection = news_db.create_connection(news_db.DB_FILE)
     while True:
-        time.sleep(1)
-        current_time = datetime.now().strftime("%H:%M")
-        if TIME_TO_SEND_START < current_time < TIME_TO_SEND_END:
-            logging.info(f"Time: {current_time}. Time to send has come !")
-            for news in news_db.send_all_news(data_from_db):
-                send_news_to_telegram(news)
-                time.sleep(300)
+        if db_connection:
+            time.sleep(1)
+            current_time = datetime.now().strftime("%H:%M")
+            if TIME_TO_SEND_START < current_time < TIME_TO_SEND_END:
+                logging.info("Time to send has come !")
+                data_from_db = news_db.send_all_news(db_connection)
+                if len(data_from_db) == 0:
+                    logging.warning("Database is empty !")
+                else:
+                    for news in data_from_db:
+                        send_news_to_telegram(news)
+                        time.sleep(3)
+            else:
+                logging.info("Still waiting to send.")
         else:
-            logging.info(f"Time: {current_time}. Still waiting to send.")
+            logging.error("Connection to db is not exist !")

docker-compose.yaml

@@ -4,6 +4,7 @@ services:
   news_db:
     container_name: news_db
     image: h0d0user/news_db:latest
+    pull_policy: always
     restart: always
     network_mode: host
     volumes:
@@ -13,6 +14,7 @@ services:
   truth_seeker:
     container_name: truth_seeker
     image: h0d0user/truth_seeker:latest
+    pull_policy: always
     restart: always
     network_mode: host
     depends_on:
@@ -24,6 +26,7 @@ services:
   datapath:
     container_name: datapath
     image: h0d0user/datapath:latest
+    pull_policy: always
     restart: always
     network_mode: host
     depends_on:

news_db.py

@@ -7,9 +7,9 @@
 
 import dynaconfig
 
-TIME_TO_PURGE = "00:00"
-
-DB_FILE = Path(f"/mnt/{dynaconfig.settings['DB_NAME']}")
+# Timing and db name loaded from settings.toml
+TIME_TO_PURGE = dynaconfig.settings["TIMINIGS"]["TIME_TO_PURGE"]
+DB_FILE = Path(f"{dynaconfig.settings['DB']['DB_NAME']}")
 
 # SQL queries
 CREATE_TABLE_SQL = """
@@ -121,19 +121,20 @@ def delete_all_news(conn):
 
 
 if __name__ == "__main__":
-    conn = create_connection(DB_FILE)
-    if DB_FILE.exists():
-        create_table(conn, CREATE_TABLE_SQL)
-
-    while True:
-        CURRENT_TIME = datetime.now().strftime("%H:%M")
-        time.sleep(1)
-        if conn is not None:
-            send_all_news(conn)
-            if CURRENT_TIME == TIME_TO_PURGE:
-                logging.info(f"Time: {CURRENT_TIME}. Time to purge has come !")
-                delete_all_news(conn)
-            else:
-                logging.info(f"Time: {CURRENT_TIME}. Still waiting for purging.")
-        else:
-            logging.error("Error! Cannot create the database connection.")
+    try:
+        conn = create_connection(DB_FILE)
+        if DB_FILE.exists():
+            create_table(conn, CREATE_TABLE_SQL)
+
+        while True:
+            CURRENT_TIME = datetime.now().strftime("%H:%M")
+            time.sleep(1)
+            if conn is not None:
+                send_all_news(conn)
+                if CURRENT_TIME == TIME_TO_PURGE:
+                    logging.info("Time to purge has come !")
+                    delete_all_news(conn)
+                else:
+                    logging.info("Still waiting for purging.")
+    except sqlite3.Error as sql_err:
+        logging.error(f"Cannot create the database connection. Error: {sql_err}")

requirements-dp.txt

@@ -1,3 +1,4 @@
+newsapi-python
 requests~=2.28.1
 dynaconf~=3.1.11
 dynaconfig~=0.4

requirements.txt

@@ -1,2 +1,6 @@
 black
-isort
+isort
+newsapi-python
+requests~=2.28.1
+dynaconf~=3.1.11
+dynaconfig~=0.4

truth_seeker.py

@@ -7,14 +7,14 @@
 import dynaconfig
 import news_db
 
-# Load variables from settings.toml
-API_KEY = dynaconfig.settings["API_KEY"]
-QUERY = dynaconfig.settings["QUERY"]
+# News API params loaded from settings.toml
+API_KEY = dynaconfig.settings["NEWS_API"]["API_KEY"]
+QUERY = dynaconfig.settings["NEWS_API"]["QUERY"]
+LANGUAGE = dynaconfig.settings["NEWS_API"]["LANGUAGE"]
+TIME_TO_SEARCH = dynaconfig.settings["TIMINIGS"]["TIME_TO_SEARCH"]
 
 newsapi = NewsApiClient(api_key=API_KEY)
 
-TIME_TO_SEARCH = "11:10"
-
 
 # Logging
 logging.basicConfig(
@@ -37,10 +37,12 @@ def fetch_info() -> dict:
     # Use to get news from yesterday to current time
     yesterday = date.today() - timedelta(days=1)
     try:
+        logging.info(
+            f"Searching for {QUERY}; most popular; from: {yesterday}; language: {LANGUAGE}"
+        )
         result = newsapi.get_everything(
-            q=QUERY, sort_by="popularity", from_param=yesterday
+            q=QUERY, sort_by="popularity", from_param=yesterday, language=LANGUAGE
         )
-        logging.info(f"Searching for {QUERY}")
         return result
     except ConnectionError as con_err:
         logging.error(con_err)
@@ -74,7 +76,7 @@ def load_to_db(fetch_info: dict):
                 tuple(article_list),
             )
     else:
-        logging.warning("Empty response from API.")
+        logging.warning("Empty response from News API.")
         raise IndexError
 
 
@@ -84,10 +86,10 @@ def load_to_db(fetch_info: dict):
         time.sleep(1)
         # Pull too much info
         if CURRENT_TIME == TIME_TO_SEARCH:
-            logging.info(f"Time: {CURRENT_TIME}. Time to search has come !")
+            logging.info("Time to search has come !")
             try:
                 load_to_db(fetch_info())
             except BaseException as base_err:
                 logging.error(base_err)
         else:
-            logging.info(f"Time: {CURRENT_TIME}. Still waiting for searching.")
+            logging.info("Still waiting for searching.")