Harden document upload path and enforce unique documentID

docs/Document-Flow.md

@@ -1,40 +1,28 @@
 # Document Flow
 
-How a credential `Document` moves through Verchain, from the API upload to the ledger and across the P2P network.
-
-## The `Document` data model
-
-Defined in `source/models/Models.py`
-
-```Python
-class Document(BaseModel):
-    model_config = ConfigDict(ser_json_bytes="base64", val_json_bytes="base64")
-
-    documentTitle: str | None = None
-    documentFormat: str | None = None
-    documentContentSize: int = 0
-    documentHash: str = ""
-
-    @model_validator(mode="before")
-    @classmethod
-    def validateLegacyDocumentContent(cls, value: Any) -> Any:
-        if isinstance(value, dict):
-            content = value.get("documentContent")
-            if isinstance(content, str):
-                try:
-                    base64.b64decode(content, altchars=b"-_", validate=True)
-                except ValueError as error:
-                    raise ValueError("Stored document content is not valid base64.") from error
-        return value
+How a credential document moves through Verchain, from API upload to the ledger and across the P2P network.
+
+## The `Document` record
+
+The `Document` model in `source/models/Models.py` stores the document title, file format, content size, and hash. The uploaded PDF bytes are used for validation and hashing, but are not stored in the model or ledger. As a result, ledger and P2P JSON contain document metadata and the digest, not the uploaded content.
+
+The `documentHash` is SHA-256 over the uploaded bytes, the format suffix, and the credential identifier:
+
+```python
+documentHash = sha256(
+    content
+    + b":"
+    + documentFormat.encode("utf-8")
+    + b":"
+    + str(documentID).encode("utf-8")
+)
 ```
 
-- `Document` uses `ser_json_bytes="base64"` / `val_json_bytes="base64"`, so raw bytes become base64 whenever a block is dumped in JOSN (ledger file, gossip, chain sync, `GET /chain`).
-- A `before` validator rejects stored content that is not valid base64 (represented as ledger corruption on load).
-- The `documentHash` is computed by `DocumentHashing` class (`services/documents.py`). The `CID` is computed in `Credential.__post_init__`.
+The credential ID (`CID`) is then computed from the document hash and `documentID`.
 
 ## Components on the document path
 
-1. API router (`POST /register`) - The entry point of a document, the document is represented as `FastAPI.UploadFile`.
-2. `DocumentReceiveer` - It takes the `UploadFile` model that contains info about the document, converts it into `Document` model (`source/models/Models.py`).
-3. `DocumentValidation` - Takes the generated `Document`, checks if it meets the `CredentialConstraints` configs.
-4. `DocumentHashing` - If the document meets the document constraints configs, the `DocumentHashing` hashes the document so it is ready to be assigned to a `Block`.
+1. `POST /register` and `POST /check` receive the PDF as a multipart `UploadFile`.
+2. `DocumentReceiver` reads at most the configured upload limit plus one byte, validates the file, and creates the `Document` record.
+3. `DocumentValidation` checks the size and supported format, requires a PDF header at the start of the file, and rejects encrypted PDFs.
+4. `DocumentHashing` hashes the file bytes and credential ID before the credential is added to a block.

docs/Hashing-in-Verchain.md

@@ -36,11 +36,15 @@ AUTHID = sha256(f"{authorityName}:{authorityID}")
 
 The `CID` is the most important ID, and it is build out of the following data:
 
-- The `documentHash` - Generated by hashing the content of the document and the document format only.
+- The `documentHash` - Generated from the document bytes, its format, and the `documentID`:
+
+```python
+documentHash = sha256(content + b":" + documentFormat + b":" + str(documentID))
+```
 - The `documentID` & document itself - Passed from the authority when checking the ownership.
 
 ```python
-CID = sha246(f"{documentHash}:{documentID}")
+CID = sha256(f"{documentHash}:{documentID}")
 ```
 
 ### The Holder Credential ID (`CHID`)

source/API/APIServices.py

@@ -8,11 +8,12 @@
 from models.Models import NodeMetadata, Credential, Authority, Block, CHID, Query, Response, User, Document
 from services.verifier import Verifier
 from utils.blocks.blockManager import BlockManager
-from utils.chain.documents import DocumentReceiver
+from services.documents import DocumentReceiver
 from validation.chain_validation import ChainValidation
 from errors import DuplicateBlockError, InvalidChainError
 from errors.holderValidationErrors import ConflictingCredentialError
 from errors.APIErrors import APIError
+from errors.credentialErrors import DuplicateCredentialIDError
 from validation.inputValidation import InputValidation
 from configs.baseConfigs import CredentialConstraints
 
@@ -111,14 +112,17 @@ def processBlockRegisterationRequest(self, registerationRequest: APIRegisteratio
 
         try:
             result: Block | Exception | None = self.peer.registerBlock(block)
-        except (DuplicateBlockError, InvalidChainError, ConflictingCredentialError, ValidationError) as error:
+        except (DuplicateBlockError, DuplicateCredentialIDError, InvalidChainError, ConflictingCredentialError, ValidationError) as error:
             result = error
         if result is None:
             return APIError(error = "Block-building-fail", message = "Cannor build the block.")
 
         if isinstance(result, DuplicateBlockError):
             return APIError(error="Block-duplication-error", message="Block already exists.")
 
+        elif isinstance(result, DuplicateCredentialIDError):
+            return APIError(error="Duplicate-Credential-ID", message=str(result))
+
         elif isinstance(result, InvalidChainError):
             return APIError(error = "Invalid-Chain", message = "Chain integrity failed; chain sync requested.")
 
@@ -161,4 +165,3 @@ def sendChain(node: NodeMetadata, blockManager: BlockManager) -> ChainModel:
         return ChainModel(sender=node, chain=blockManager.ledger.blocks)
 
 
-

source/API/router.py

@@ -8,8 +8,8 @@
 from API.APIServices import APICommunication
 from errors.APIErrors import APIError
 from configs.handler import configs
-# from utils.chain.documents import DocumentReceiver
 from services.documents import DocumentReceiver
+from configs.baseConfigs import CredentialConstraints
 
 
 def buildRouter(peer: Peer) -> APIRouter:
@@ -43,8 +43,8 @@ async def register(
             "credential.document": document,
         })
         try:
-            documentReceiver = DocumentReceiver(document=document)
-            receivedDocument = await documentReceiver.receive(uploadedFile=document)
+            documentReceiver = DocumentReceiver(constraints=CredentialConstraints())
+            receivedDocument = await documentReceiver.receive(uploadedFile=document, documentID=documentID)
         except ValueError as error:
             raise HTTPException(status_code=400, detail=str(error)) from error
 
@@ -79,8 +79,8 @@ async def check(
               issuerID: Annotated[int, Form(alias="issuerID")]):
 
         try:
-            documentReceiver = DocumentReceiver(document=document)
-            receivedDocument = await documentReceiver.receive(uploadedFile=document)
+            documentReceiver = DocumentReceiver(constraints=CredentialConstraints())
+            receivedDocument = await documentReceiver.receive(uploadedFile=document, documentID=documentID)
         except ValueError as error:
             raise HTTPException(status_code=400, detail=str(error)) from error
 
@@ -97,7 +97,10 @@ async def check(
         result: Response | APIError = communication.processVerificationRequest(payload)
 
         if isinstance(result, APIError):
-            return result
+            raise HTTPException(
+                status_code=404,
+                detail={"error": result.error, "message": result.message},
+            )
 
         return result
 

source/errors/__init__.py

@@ -24,4 +24,5 @@
 
 from errors.chainSyncErrors import (
     ReceivedChainIsInvalid
-)
+)
+from errors.credentialErrors import DuplicateCredentialIDError

source/errors/credentialErrors.py

@@ -0,0 +1,4 @@
+class DuplicateCredentialIDError(Exception):
+    def __init__(self, documentID: int) -> None:
+        self.documentID = documentID
+        super().__init__(f"Credential documentID {documentID} is already registered.")

source/errors/documents.py

@@ -20,5 +20,5 @@ def __init__(self, supported_formats: set[str] | None = None) -> None:
 class InvalidPDFContent(Exception):
     def __init__(self, filename: str) -> None:
         super().__init__(
-            f"The uploaded file: `{filename}` has ivalid content."
+            f"The uploaded file: `{filename}` has invalid content."
         )

source/services/documents.py

@@ -14,7 +14,6 @@
 import hashlib
 from pathlib import Path
 from fastapi import UploadFile
-from pymupdf import pymupdf
 
 from configs.baseConfigs import CredentialConstraints
 from models.Models import Document
@@ -26,20 +25,20 @@
 class DocumentReceiver:
     """Validate an uploaded credential and return its content digest."""
 
-    def __init__(self, document: UploadFile, constraints: CredentialConstraints | None = None) -> None:
-        self.constraints = constraints or CredentialConstraints()
-        self.document = document
+    def __init__(self, constraints: CredentialConstraints) -> None:
+        self.constraints = constraints
+
 
-    async def receive(self, uploadedFile: UploadFile) -> Document:
+    async def receive(self, uploadedFile: UploadFile, documentID: int) -> Document:
         "Builds the ``Document`` model from the ``UploadFile``"
 
         filename = uploadedFile.filename or ""
         fileFormat = Path(filename).suffix.lower()
-        content = await self.read()
+        content = await self.read(uploadedFile)
 
         document = Document(
             documentTitle="Credential document",
-            documentFormat=fileFormat,
+            documentFormat=fileFormat.lower(),
             documentContentSize=len(content),
         )
 
@@ -49,52 +48,39 @@ async def receive(self, uploadedFile: UploadFile) -> Document:
             raise ValueError(str(error)) from error
 
 
-
-
-        documentHashBuilder = DocumentHashing(document=document, content=content)
+        documentHashBuilder = DocumentHashing(document=document, content=content, documentID=documentID)
         hashedDocument: Document = documentHashBuilder.hash()
 
         return hashedDocument
 
-
-
 
 
-    async def read(self) -> bytes:
+    async def read(self, file: UploadFile) -> bytes:
         "Reads the received document and returns it's content (bytes)."
-        if self.document is None:
-            return b""
-
-        file_handle = getattr(self.document, "file", None)
-        if file_handle is None:
-            return b""
-
         try:
-            file_handle.seek(0)
-            return file_handle.read()
+            await file.seek(0)
+            return await file.read(self.constraints.maxUploadSize + 1)
         finally:
-            file_handle.seek(0)
+            await file.seek(0)
 
 
 
 
 
 class DocumentHashing:
-    def __init__(self, document: Document, content: bytes) -> None:
+    def __init__(self, document: Document, content: bytes, documentID: int) -> None:
         self.document = document
         self.content = content
+        self.ID = str(documentID).encode()
 
     def hash(self) -> Document:
-        # payload = self.document.documentContent or b""
         file_format = (self.document.documentFormat or "").encode("utf-8")
         documentHash = hashlib.sha256(
-            self.content + b":" + file_format
+            self.content + b":" + file_format + b":" + self.ID
         ).hexdigest()
 
         self.document.documentHash = documentHash
 
         return self.document
 
 
-
-

source/services/ledger.py

@@ -21,6 +21,7 @@
     )
 
 from errors.blockErrors import BlockIntegrationError
+from errors.credentialErrors import DuplicateCredentialIDError
 
 from configs import settings, Performance
 
@@ -41,6 +42,7 @@ def __init__(self, filePath: Path) -> None:
         self.usersByUsername: dict[str, User] = {}
 
         self.credentials: dict[str, Credential] = {}
+        self.credentialsByDocumentID: dict[int, Credential] = {}
         self.issuers: dict[int, Authority] = {}
 
         self.miner = Miner()
@@ -110,6 +112,12 @@ def __createFileIfnotExist(self) -> None:
 
     def insertBlock(self, block: Block) -> Block:
         with self._lock:
+            credential = block.data.credential
+            if (
+                credential.document.documentHash
+                and credential.documentID in self.credentialsByDocumentID
+            ):
+                raise DuplicateCredentialIDError(credential.documentID)
 
             blockAsDict = json.loads(Block.model_dump_json(block))
             blockChid = blockAsDict.get("data", {}).get("chid")
@@ -132,7 +140,10 @@ def insertBlock(self, block: Block) -> Block:
             self.blocks.append(blockAsDict)
 
             self.users[block.data.user.nationalNumber] = block.data.user
-            self.credentials[block.data.credential.document.documentHash] = block.data.credential
+            documentHash = credential.document.documentHash
+            if documentHash:
+                self.credentials[documentHash] = credential
+                self.credentialsByDocumentID[credential.documentID] = credential
             self.issuers[block.data.issuer.businessID] = block.data.issuer
 
             self.__writeBlockToLedger(blockAsDict)
@@ -189,6 +200,7 @@ def loadFromLedger(self) -> None:
         self.usersByNationalNumber.clear()
         self.usersByUsername.clear()
         self.credentials.clear()
+        self.credentialsByDocumentID.clear()
         self.issuers.clear()
 
         try:
@@ -197,7 +209,14 @@ def loadFromLedger(self) -> None:
                 self.usersByNationalNumber[block["data"]["user"]["nationalNumber"]] = User.model_validate(block["data"]["user"])
 
                 documentHash = block["data"]["credential"]["document"]["documentHash"]
-                self.credentials[documentHash] = Credential.model_validate(block["data"]["credential"])
+                if documentHash:
+                    credential = Credential.model_validate(block["data"]["credential"])
+                    if credential.documentID in self.credentialsByDocumentID:
+                        raise LedgerCorruptError(
+                            f"Credential documentID {credential.documentID} is duplicated in the ledger."
+                        )
+                    self.credentials[documentHash] = credential
+                    self.credentialsByDocumentID[credential.documentID] = credential
                 self.issuers[block["data"]["issuer"]["businessID"]] = Authority.model_validate(block["data"]["issuer"])
         except ValidationError as error:
             raise LedgerCorruptError(str(self.filePath)) from error
@@ -217,6 +236,7 @@ def updateLedger(self, newLedger: list) -> None:
 
         self.users: dict[int, User] = {}
         self.credentials: dict[str, Credential] = {}
+        self.credentialsByDocumentID: dict[int, Credential] = {}
         self.issuers: dict[int, Authority] = {}
 
         self.loadFromLedger()
@@ -236,6 +256,9 @@ def findUser(self, nationalNumber: int, username: str) -> User | None:
     def findCredential(self, documentHash: str) -> Credential | None:
         return self.credentials.get(documentHash, None)
 
+    def findCredentialByDocumentID(self, documentID: int) -> Credential | None:
+        return self.credentialsByDocumentID.get(documentID)
+
     def findIssuer(self, issuerID: int, issuerName: str) -> Authority | None:
         issuer = self.issuers.get(issuerID, None)
 
@@ -250,4 +273,3 @@ def findIssuer(self, issuerID: int, issuerName: str) -> Authority | None:
 
 
 
-

source/services/peer.py

@@ -14,6 +14,7 @@
 from utils.nodeStorageManager import NodeStorageManager
 from utils.blocks.blockManager import BlockManager
 from errors.holderValidationErrors import ConflictingCredentialError
+from errors.credentialErrors import DuplicateCredentialIDError
 from services.ledger import Ledger
 from services.network import Network
 from models.network import NetworkContext
@@ -165,6 +166,9 @@ def registerBlock(self, block: Block) -> Block | None | Exception:
         except DuplicateBlockError:
             return DuplicateBlockError(block.data.chid)
 
+        except DuplicateCredentialIDError:
+            return DuplicateCredentialIDError(block.data.credential.documentID)
+
         except ConflictingCredentialError:
             return ConflictingCredentialError(
                 nationalNumber=block.data.user.nationalNumber,