Update CI; change .secrets.toml to settings.toml; more info about funcs; small fixes

CoolCoderCarl · CoolCoderCarl · commit 355098e8b89a · 2022-11-06T15:18:29.000+03:00
diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -103,52 +103,35 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: windows-latest
-            TARGET: windows
-            CMD_BUILD: python -m PyInstaller --clean --workpath /tmp/build --specpath /tmp -F ./app/truth_seeker.py
-            OUT_FILE_NAME: truth_seeker.exe
-            ASSET_MIME: application/vnd.microsoft.portable-executable
           - os: ubuntu-latest
-            TARGET: ubuntu
-            CMD_BUILD: python -m PyInstaller --clean --workpath /tmp/build --specpath /tmp -F ./app/truth_seeker.py
-            OUT_FILE_NAME: truth_seeker
-            ASSET_MIME: application/x-binary
+            SERVICENAME: news_db
+            CONTAINERFILE: Containerfile-db
+          - os: ubuntu-latest
+            SERVICENAME: truth_seeker
+            CONTAINERFILE: Containerfile-ts
+          - os: ubuntu-latest
+            SERVICENAME: datapath
+            CONTAINERFILE: Containerfile-dp
 
     steps:
 
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
-          python-version: 3.9
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-
-      - name: Build with pyinstaller for ${{ matrix.TARGET }}
-        run: ${{ matrix.CMD_BUILD }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Upload Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ matrix.OUT_FILE_NAME }}
-          path: ./dist/${{ matrix.OUT_FILE_NAME }}
-          retention-days: 1
+      - name: Checkout code
+        uses: actions/checkout@v2
 
-      - name: Upload Release Asset
-        id: upload-release-asset
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        id: push_image
+        uses: docker/build-push-action@v2
         with:
-          upload_url: ${{ needs.release.outputs.get_release_url }}
-          asset_path: ./dist/${{ matrix.OUT_FILE_NAME }}
-          asset_name: ${{ matrix.OUT_FILE_NAME }}
-          asset_content_type: ${{ matrix.ASSET_MIME }}
+          context: .
+          push: true
+          file: ${{ matrix.CONTAINERFILE }}
+          tags: h0d0user/${{ matrix.SERVICENAME }}:latest
 
       - name: Notify if failure
         if: ${{ failure() }}
@@ -186,6 +169,11 @@ jobs:
 
             Branch: ${{ github.ref }}
 
+            <b>Pushed image:</b> <code>docker pull h0d0user/news_db:latest</code>,
+            <code>docker pull h0d0user/truth_seeker:latest</code>,
+            <code>docker pull h0d0user/datapath:latest</code>
+            See changes: https://github.com/${{ github.repository }}/commit/${{ github.sha }}
+
             <b>Release URL:</b> https://github.com/${{ github.repository }}/releases/tag/${{ needs.prepare.outputs.get_current_tag }}
 
             See changes: https://github.com/${{ github.repository }}/commit/${{ github.sha }}
diff --git a/.gitignore b/.gitignore
@@ -129,7 +129,7 @@ dmypy.json
 .pyre/
 
 # Secret file
-.secrets.toml
+settings.toml
 
 # Database files
 *.db
diff --git a/datapath.py b/datapath.py
@@ -26,6 +26,11 @@
 
 
 def send_news_to_telegram(message):
+    """
+    Send messages from db to telegram
+    :param message:
+    :return:
+    """
     try:
         response = requests.post(API_URL, json={"chat_id": CHAT_ID, "text": message})
         if response.status_code == 200:
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -3,34 +3,34 @@ version: '3'
 services:
   news_db:
     container_name: news_db
-    image: h0d0user/news_db:v0.0.49
+    image: h0d0user/news_db:latest
     restart: always
     network_mode: host
     volumes:
       - "shared-volume:/mnt/"
-      - ".secrets.toml:/opt/.secrets.toml"
+      - "settings.toml:/opt/settings.toml"
 
   truth_seeker:
     container_name: truth_seeker
-    image: h0d0user/truth_seeker:v0.0.49
+    image: h0d0user/truth_seeker:latest
     restart: always
     network_mode: host
     depends_on:
       - "news_db"
     volumes:
       - "shared-volume:/mnt/"
-      - ".secrets.toml:/opt/.secrets.toml"
+      - "settings.toml:/opt/settings.toml"
 
   datapath:
     container_name: datapath
-    image: h0d0user/datapath:v0.0.49
+    image: h0d0user/datapath:latest
     restart: always
     network_mode: host
     depends_on:
       - "news_db"
     volumes:
       - "shared-volume:/mnt/"
-      - ".secrets.toml:/opt/.secrets.toml"
+      - "settings.toml:/opt/settings.toml"
 
 volumes:
   shared-volume:
diff --git a/dynaconfig.py b/dynaconfig.py
@@ -8,7 +8,7 @@
 )
 
 settings = Dynaconf(
-    settings_files=[".secrets.toml"],
+    settings_files=["settings.toml"],
 )
 
 if __name__ == "__main__":
diff --git a/news_db.py b/news_db.py
@@ -5,11 +5,13 @@
 from pathlib import Path
 from sqlite3 import Error
 
-TIME_TO_PURGE = "00:00"
+import dynaconfig
 
+TIME_TO_PURGE = "00:00"
 
-DB_FILE = Path("/mnt/test.db")
+DB_FILE = Path(f"/mnt/{dynaconfig.settings['DB_NAME']}")
 
+# SQL queries
 CREATE_TABLE_SQL = """
 CREATE TABLE IF NOT EXISTS news (
 author TEXT,
@@ -46,6 +48,11 @@
 
 
 def create_connection(db_file: Path):
+    """
+    Create db file
+    :param db_file: path to db file to create
+    :return:
+    """
     try:
         conn = sqlite3.connect(db_file)
         logging.info("Connection created successfully !")
@@ -72,6 +79,7 @@ def create_table(conn, create_table_query):
 def insert_into(conn, data: tuple):
     """
     Insert data to base
+    This is the load step in ETL pipeline
     :param conn: Connection to the SQLite database
     :param data:
     :return:
diff --git a/truth_seeker.py b/truth_seeker.py
@@ -7,12 +7,10 @@
 import dynaconfig
 import news_db
 
+# Load variables from settings.toml
 API_KEY = dynaconfig.settings["API_KEY"]
 QUERY = dynaconfig.settings["QUERY"]
 
-# Use to get news from yesterday to current time
-YESTERDAY = date.today() - timedelta(days=1)
-
 newsapi = NewsApiClient(api_key=API_KEY)
 
 TIME_TO_SEARCH = "11:10"
@@ -31,9 +29,16 @@
 
 
 def fetch_info() -> dict:
+    """
+    Fetch info from API according to query in settings.toml
+    This is the extraction step in ETL pipeline
+    :return:
+    """
+    # Use to get news from yesterday to current time
+    yesterday = date.today() - timedelta(days=1)
     try:
         result = newsapi.get_everything(
-            q=QUERY, sort_by="popularity", from_param=YESTERDAY
+            q=QUERY, sort_by="popularity", from_param=yesterday
         )
         logging.info(f"Searching for {QUERY}")
         return result
@@ -46,6 +51,12 @@ def fetch_info() -> dict:
 
 
 def load_to_db(fetch_info: dict):
+    """
+    Load specific info to db
+    These are the transform and load steps in ETL pipeline
+    :param fetch_info:
+    :return:
+    """
     if fetch_info:
         for article in fetch_info.get("articles"):
             article_list = []

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`	`)`
`9`	`9`
`10`	`10`	`settings = Dynaconf(`
`11`		`- settings_files=[".secrets.toml"],`
	`11`	`+ settings_files=["settings.toml"],`
`12`	`12`	`)`
`13`	`13`
`14`	`14`	`if __name__ == "__main__":`