diff --git a/.github/workflows/first_pipeline.yml b/.github/workflows/first_pipeline.yml index c899a06..ebd0d14 100644 --- a/.github/workflows/first_pipeline.yml +++ b/.github/workflows/first_pipeline.yml @@ -1,7 +1,7 @@ -name: CI +name: CI/CD Pipeline on: push: - branches: ["master"] + branches: ["master", "dev"] jobs: build_job: @@ -39,6 +39,7 @@ jobs: - name: Create .env.test file run: | echo "TEST_DATABASE_URL=postgresql+asyncpg://test_user:test_password@localhost:5432/test_db" >> .env.test + echo "TEST_REDIS_URL=redis://localhost:6379/0" >> .env.test - name: Check prod with linters run: | @@ -49,11 +50,13 @@ jobs: - name: Check prod with pytest env: DATABASE_URL: "postgresql+asyncpg://test_user:test_password@localhost:5432/test_db" + REDIS_URL: "redis://:test_password@localhost:6379/1" run: pytest deploy_job: needs: build_job runs-on: self-hosted + if: github.ref == 'refs/heads/master' steps: - name: Checkout uses: actions/checkout@v4 @@ -64,6 +67,10 @@ jobs: echo "POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}" >> .env echo "POSTGRES_DB=${{ secrets.POSTGRES_DB }}" >> .env echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" >> .env + + echo "REDIS_PASSWORD=${{ secrets.REDIS_PASSWORD }}" >> .env + echo "REDIS_DB=${{ secrets.REDIS_DB }}" >> .env + echo "REDIS_URL=${{ secrets.REDIS_URL }}" >> .env - name: Stops all containers run: docker compose down diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml index 5ba5524..41fe702 100644 --- a/.idea/dictionaries/project.xml +++ b/.idea/dictionaries/project.xml @@ -1,6 +1,7 @@ + requirepass shurlsdb diff --git a/app/api/endpoints.py b/app/api/endpoints.py index 4b35822..db840f2 100644 --- a/app/api/endpoints.py +++ b/app/api/endpoints.py @@ -1,15 +1,20 @@ -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request from fastapi.responses import RedirectResponse +from pydantic import HttpUrl +from redis import asyncio as asyncredis from slowapi import Limiter from slowapi.util import get_remote_address from sqlalchemy.ext.asyncio import AsyncSession from app.core.database import get_db -from app.schemas.url import UrlCreate, UrlInfo +from app.core.redis_db import get_redis +from app.schemas.url_scheme import UrlCreate, UrlInfo from app.services.url_service import ( - create_short_url, + generate_secret_key, get_original_url, get_short_url_stats, + increment_clicks_worker, + save_url_worker, ) router = APIRouter() @@ -20,18 +25,46 @@ @router.post("/shorten", response_model=UrlInfo) @limiter.limit("10/minute") async def shorten( - request: Request, scheme: UrlCreate, db: AsyncSession = Depends(get_db) + request: Request, + scheme: UrlCreate, + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + redis_client: asyncredis.Redis = Depends(get_redis), ): - created_url = await create_short_url(db, str(scheme.target_url)) - return created_url + secret_key = generate_secret_key() + + background_tasks.add_task( + save_url_worker, + secret_key=secret_key, + target_url=str(scheme.target_url), + redis_client=redis_client, + db=db, + ) + + return UrlInfo( + target_url=HttpUrl(scheme.target_url), + secret_key=secret_key, + clicks=0, + is_active=True, + ) @router.get("/{secret_key}") -async def redirect(secret_key: str, db: AsyncSession = Depends(get_db)): - original_url = await get_original_url(db, secret_key) +async def redirect( + secret_key: str, + background_tasks: BackgroundTasks, + db: AsyncSession = Depends(get_db), + redis_client: asyncredis.Redis = Depends(get_redis), +): + original_url = await get_original_url( + redis_client=redis_client, db=db, secret_key=secret_key + ) if not original_url: raise HTTPException(status_code=404) - return RedirectResponse(original_url.target_url) + + background_tasks.add_task(increment_clicks_worker, secret_key=secret_key, db=db) + + return RedirectResponse(original_url) @router.get("/stats/{secret_key}") diff --git a/app/core/config.py b/app/core/config.py index d7f7250..257dde5 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -11,6 +11,7 @@ class AppBaseSettings(BaseSettings): ) database_url: str + redis_url: str settings = AppBaseSettings() diff --git a/app/core/redis_db.py b/app/core/redis_db.py new file mode 100644 index 0000000..ed17026 --- /dev/null +++ b/app/core/redis_db.py @@ -0,0 +1,5 @@ +from fastapi import Request + + +async def get_redis(request: Request): + return request.app.state.redis diff --git a/app/main.py b/app/main.py index 5793796..3ed7ff9 100644 --- a/app/main.py +++ b/app/main.py @@ -1,11 +1,26 @@ +from contextlib import asynccontextmanager + +import redis.asyncio as asyncredis import uvicorn from fastapi import FastAPI from slowapi import _rate_limit_exceeded_handler from slowapi.errors import RateLimitExceeded from app.api.endpoints import limiter, router +from app.core.config import settings + + +@asynccontextmanager +async def lifespan(app: FastAPI): + r_client = asyncredis.from_url(settings.redis_url, decode_responses=True) + app.state.redis = r_client + + yield + + await app.state.redis.aclose() + -app = FastAPI() +app = FastAPI(lifespan=lifespan) app.include_router(router) diff --git a/app/schemas/url.py b/app/schemas/url_scheme.py similarity index 100% rename from app/schemas/url.py rename to app/schemas/url_scheme.py diff --git a/app/services/url_service.py b/app/services/url_service.py index 530362a..c29b6ca 100644 --- a/app/services/url_service.py +++ b/app/services/url_service.py @@ -1,30 +1,60 @@ +import logging import secrets -from sqlalchemy import select +from redis import asyncio as asyncredis +from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from app.models.url import Url -async def create_short_url(db: AsyncSession, target_url: str) -> Url: - secret = secrets.token_urlsafe(8) - url = Url(secret_key=secret, target_url=target_url) - db.add(url) - await db.commit() - await db.refresh(url) +def generate_secret_key(): + return secrets.token_urlsafe(8) - return url +async def increment_clicks_worker(secret_key: str, db: AsyncSession): + try: + await db.execute( + update(Url) + .where(Url.secret_key == secret_key) + .values(clicks=Url.clicks + 1) + ) + await db.commit() + except Exception as e: + logging.error(f"Failed to increment clicks for {secret_key}: {e}") + + +async def save_url_worker( + secret_key: str, target_url: str, redis_client, db: AsyncSession +): + try: + new_url = Url(secret_key=secret_key, target_url=target_url) + db.add(new_url) + await db.commit() + cache_key = f"url_{secret_key}" + await redis_client.set(cache_key, target_url, ex=86400) + + except Exception as e: + logging.error(f"Ошибка при сохранении {secret_key}: {e}") + + +async def get_original_url( + redis_client: asyncredis.Redis, db: AsyncSession, secret_key: str +) -> str: + url_from_cache = await redis_client.get(f"url_{secret_key}") + if url_from_cache: + return url_from_cache -async def get_original_url(db: AsyncSession, secret_key: str): result = await db.execute(select(Url).where(Url.secret_key == secret_key)) url_object = result.scalars().first() - if url_object: - url_object.clicks = Url.clicks + 1 - await db.commit() - await db.refresh(url_object) - return url_object + if not url_object: + return "" + + final_url = str(url_object.target_url) + await redis_client.set(f"url_{secret_key}", final_url, ex=86400) + + return final_url async def get_short_url_stats(db: AsyncSession, secret_key: str): diff --git a/docker-compose.yml b/docker-compose.yml index ddee6b4..ee0d482 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,13 @@ services: web: build: . depends_on: - - db + db: + condition: service_healthy + redis: + condition: service_healthy environment: DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} db: image: postgres:17 @@ -13,9 +17,27 @@ services: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 5 volumes: - postgres_data:/var/lib/postgresql/data + redis: + image: redis:alpine + restart: always + command: redis-server --requirepass ${REDIS_PASSWORD} + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_DB: ${REDIS_DB} + healthcheck: + test: ["CMD-SHELL", "redis-cli -a $${REDIS_PASSWORD} ping | grep PONG"] + interval: 5s + timeout: 3s + retries: 5 + nginx: build: ./nginx ports: diff --git a/pytest.ini b/pytest.ini index 28213f0..f2c95d5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,4 @@ pythonpath = . asyncio_mode = auto asyncio_default_fixture_loop_scope = function +asyncio_default_test_loop_scope = session \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7ee746e..e17e5e3 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/tests/conftest.py b/tests/conftest.py index bd00eac..2aef97f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,11 +5,13 @@ import pytest import pytest_asyncio from dotenv import load_dotenv +from fakeredis import FakeAsyncRedis from httpx import ASGITransport, AsyncClient from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool from app.core.database import Base, get_db +from app.core.redis_db import get_redis from app.main import app if sys.platform == "win32": @@ -45,9 +47,20 @@ async def override_db(): yield session +@pytest_asyncio.fixture() +async def mock_redis(): + client = FakeAsyncRedis(decode_responses=True) + yield client + await client.flushall() + + @pytest_asyncio.fixture(scope="function") -async def client(): +async def client(mock_redis): + async def override_redis(): + yield mock_redis + app.dependency_overrides[get_db] = override_db + app.dependency_overrides[get_redis] = override_redis async with AsyncClient( transport=ASGITransport(app=app), base_url="http://test" diff --git a/tests/test_api.py b/tests/test_api.py index 1a70247..26d0f46 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -31,23 +31,23 @@ async def test_404_not_found(client): async def test_zero_link_clicks(client): response = await client.post("/shorten", json={"target_url": "https://youtube.com"}) response_data = response.json() - + print(response_data) url_stats = await client.get(f"/stats/{response_data['secret_key']}") stats_data = url_stats.json() - + print(stats_data) assert stats_data["clicks"] == 0 async def test_multiple_link_clicks(client): response = await client.post("/shorten", json={"target_url": "https://youtube.com"}) response_data = response.json() - + print(response_data) for i in range(10): await client.get(f"/{response_data["secret_key"]}", follow_redirects=False) url_stats = await client.get(f"/stats/{response_data['secret_key']}") stats_data = url_stats.json() - + print(stats_data) assert stats_data["clicks"] == 10