Feature/sub corpus

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "1.5.2"
+version = "1.5.3"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}

tests/test_corpus_related.py

@@ -37,36 +37,46 @@ def test_creates_and_reads_corpus(self):
             source_name="Corpus Test",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.7,
             category_id=category.id,
+            main_url="https://example.org",
         )
         self.session.add(corpus)
         self.session.commit()
         result = self.session.query(Corpus).filter_by(source_name="Corpus Test").first()
         self.assertIsNotNone(result)
-        self.assertEqual(float(result.binary_treshold), 0.7)
+        self.assertEqual(result.main_url, "https://example.org")
 
-    def test_updates_corpus_binary_treshold(self):
+    def test_creates_and_reads_parent_corpus(self):
         category = Category(id=uuid.uuid4(), title="Category Test")
         self.session.add(category)
         self.session.commit()
+        corpus_parent_id = uuid.uuid4()
         corpus = Corpus(
-            id=uuid.uuid4(),
+            id=corpus_parent_id,
             source_name="Corpus Test",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.7,
             category_id=category.id,
+            main_url="https://example.org",
+        )
+        corpus2 = Corpus(
+            id=uuid.uuid4(),
+            parent_corpus_id=corpus_parent_id,
+            source_name="Sub Corpus Test",
+            is_fix=True,
+            is_active=True,
+            category_id=category.id,
+            main_url="https://example.org/sub",
         )
         self.session.add(corpus)
+        self.session.add(corpus2)
         self.session.commit()
-        corpus.binary_treshold = 0.9
-        self.session.commit()
-        updated_corpus = (
-            self.session.query(Corpus).filter_by(source_name="Corpus Test").first()
+        result = (
+            self.session.query(Corpus).filter_by(source_name="Sub Corpus Test").first()
         )
-        self.assertIsNotNone(updated_corpus)
-        self.assertEqual(float(updated_corpus.binary_treshold), 0.9)
+        self.assertIsNotNone(result)
+        self.assertEqual(result.main_url, "https://example.org/sub")
+        self.assertEqual(result.parent_corpus_id, corpus_parent_id)
 
     def test_deletes_corpus(self):
         category = Category(id=uuid.uuid4(), title="Category Test")
@@ -77,7 +87,6 @@ def test_deletes_corpus(self):
             source_name="Corpus Test",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.7,
             category_id=category.id,
         )
         self.session.add(corpus)

tests/test_create_database.py

@@ -87,7 +87,6 @@ def test_crud_document_and_bookmark(self):
             source_name="Corpus Test",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=category.id,
         )
         self.session.add(corpus)

tests/test_document_related.py

@@ -196,7 +196,6 @@ def test_external_id(self):
             source_name="Test Corpus",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=category_id,
         )
         test_session.add(test_corpus)
@@ -271,7 +270,6 @@ def test_trace_in_db(self):
             source_name="Test Corpus",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=test_category.id,
         )
         test_session.add(test_corpus)
@@ -318,7 +316,6 @@ def test_none_trace_in_db(self):
             source_name="Test Corpus",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=test_category.id,
         )
         test_session.add(test_corpus)
@@ -367,7 +364,6 @@ def test_view_qty_document(self):
             source_name="Test Corpus",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=test_category.id,
         )
         test_session.add(test_corpus)
@@ -435,7 +431,6 @@ def test_error_data_quality(self):
             source_name="Test Corpus",
             is_fix=True,
             is_active=True,
-            binary_treshold=0.5,
             category_id=test_category.id,
         )
         test_session.add(test_corpus)

welearn_database/alembic/versions/5d46d7920342_support_sub_corpus.py

@@ -0,0 +1,75 @@
+"""support_sub_corpus
+
+Revision ID: 5d46d7920342
+Revises: 95ffe7afa64c
+Create Date: 2026-09-21 14:47:28.116941
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "5d46d7920342"
+down_revision: Union[str, None] = "95ffe7afa64c"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "corpus",
+        sa.Column("parent_corpus_id", sa.Uuid(), nullable=True),
+        schema="corpus_related",
+    )
+    op.create_foreign_key(
+        "corpus_parent_corpus_id_fkey",
+        "corpus",
+        "corpus",
+        ["parent_corpus_id"],
+        ["id"],
+        source_schema="corpus_related",
+        referent_schema="corpus_related",
+    )
+    op.execute("DROP VIEW grafana.corpus")
+    op.drop_column("corpus", "binary_treshold", schema="corpus_related")
+    op.execute("""
+            CREATE OR REPLACE VIEW grafana.corpus
+            AS SELECT corpus.id,
+            corpus.parent_corpus_id,
+            corpus.source_name,
+            corpus.main_url,
+            corpus.is_fix,
+            corpus.is_active,
+            corpus.category_id
+           FROM corpus_related.corpus;
+    """)
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "corpus_parent_corpus_id_fkey", "corpus", schema="corpus_related"
+    )
+    op.execute("DROP VIEW grafana.corpus")
+    op.drop_column("corpus", "parent_corpus_id", schema="corpus_related")
+    op.add_column(
+        "corpus",
+        sa.Column(
+            "binary_treshold",
+            sa.Float(),
+            nullable=False,
+            default=0.5,
+            server_default="0.5",
+        ),
+        schema="corpus_related",
+    )
+    op.execute("""
+            CREATE OR REPLACE VIEW grafana.corpus
+            AS SELECT corpus.id,
+            corpus.source_name,
+            corpus.is_fix,
+            corpus.binary_treshold
+           FROM corpus_related.corpus;
+    """)

welearn_database/data/models/corpus_related.py

@@ -19,10 +19,13 @@ class Corpus(Base):
     id: Mapped[UUID] = mapped_column(
         types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
     )
+    parent_corpus_id: Mapped[UUID | None] = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{schema_name}.corpus.id"),
+    )
     source_name: Mapped[str]
     main_url: Mapped[str | None] = mapped_column(nullable=True)
     is_fix: Mapped[bool]
-    binary_treshold: Mapped[float] = mapped_column(nullable=False, default=0.5)
     is_active: Mapped[bool]
     category_id: Mapped[UUID] = mapped_column(
         types.Uuid,

welearn_database/data/models/grafana.py

@@ -16,10 +16,13 @@ class Corpus(Base):
     __table_args__ = {"schema": schema_name}
     __read_only__ = True
 
-    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
-    source_name: Mapped[str] = mapped_column()
-    is_fix: Mapped[bool | None] = mapped_column(nullable=True)
-    binary_treshold: Mapped[float | None] = mapped_column(nullable=True)
+    id: Mapped[UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
+    parent_corpus_id: Mapped[UUID | None] = mapped_column(UUID(as_uuid=True))
+    source_name: Mapped[str]
+    main_url: Mapped[str | None] = mapped_column(nullable=True)
+    is_fix: Mapped[bool]
+    is_active: Mapped[bool]
+    category_id: Mapped[UUID | None] = mapped_column(UUID(as_uuid=True))
 
 
 class DocumentLatestState(Base):