From 02378baa80eaf99b0be86cb36e8e180eb601cde0 Mon Sep 17 00:00:00 2001
From: andreighh <61791875+andreighh@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:26:27 +0300
Subject: [PATCH 1/3] Fix UNGM scraper after site changes

  UNGM's notice search now requires an ASP.NET antiforgery token and the
  list/notice markup changed, so the scraper got no results.

  - Load the search page and POST in one requests.Session, sending the
    __RequestVerificationToken back in the RequestVerificationToken header
  - Add the new search payload fields sent by the site
  - Use consistent, current Firefox headers for GET and POST
  - Add timeouts to every UNGM request, longer backoff between retries
  - Pause 2-5s between notice requests and skip notices that fail to load
  - Parse list rows by tableCell instead of positional .contents indexes
  - Limit notice parsing to #noticeDetail (the menu reuses status-tag)
  - Tests and fixtures for the 2026 list and notice pages
---
 app/parsers/ungm.py                           |  35 ++--
 app/server_requests.py                        | 109 ++++++++----
 app/tests/parser_files/ungm_notice_2026.html  | 155 +++++++++++++++++
 .../parser_files/ungm_notice_list_2026.html   | 160 ++++++++++++++++++
 app/tests/test_ungm_parser.py                 |  51 ++++++
 5 files changed, 466 insertions(+), 44 deletions(-)
 create mode 100644 app/tests/parser_files/ungm_notice_2026.html
 create mode 100644 app/tests/parser_files/ungm_notice_list_2026.html

diff --git a/app/parsers/ungm.py b/app/parsers/ungm.py
index 40307bb..043d712 100644
--- a/app/parsers/ungm.py
+++ b/app/parsers/ungm.py
@@ -14,7 +14,9 @@
 
 from app.exceptions import UNSPCCodesNotFound
 from app.models import UNSPSCCode, WorkerLog, Tender, TenderDocument
-from app.server_requests import get_request_class
+from app.server_requests import (
+    get_request_class, GET_HEADERS, REQUEST_TIMEOUT
+)
 from scratch import settings
 
 
@@ -65,14 +67,15 @@ def parse_ungm_notice_list(html):
 
         tenders_list = []
         for tender in tenders:
+            # Columns: options, title, deadline, published, organization,
+            # type, reference, country
+            cells = tender.find_all('div', class_='tableCell', recursive=False)
+            href = cells[1].a['href'].strip() if cells[1].a else ''
             tender_dict = {
                 'published': UNGMWorker.parse_date(
-                    tender.contents[7].span.string.strip(), '%d-%b-%Y'),
-                'reference': tender.contents[13].span.string.strip(),
-                'url': (
-                    endpoint + tender.contents[3].a['href'].strip()
-                    if tender.contents[3].a['href'].strip() else ''
-                ),
+                    cells[3].span.get_text().strip(), '%d-%b-%Y'),
+                'reference': cells[6].span.get_text().strip(),
+                'url': endpoint + href if href else '',
             }
             if tender_dict['url'] == 'http://google.com':
                 # This is a bug that should be investigated
@@ -96,6 +99,9 @@ def parse_ungm_notice(html, url, codes):
             the tender is invalid.
         """
         soup = BeautifulSoup(html, 'html.parser')
+        # Restrict to the notice itself, the page menu also uses classes
+        # like "status-tag"
+        soup = soup.find(id='noticeDetail') or soup
         documents = UNGMWorker.find_by_class(soup, "lnkShowDocument", "a")
         description = UNGMWorker.find_by_class(
             soup, "ungm-list-item ungm-background", "div")
@@ -175,7 +181,11 @@ def parsed_tenders(self, tenders):
                 url = tender.url
             except AttributeError:
                 url = tender['url']
+            sleep(randint(2, 5))
             html = self.requester.get_request(url)
+            if not html:
+                logging.warning('Skipping UNGM tender %s, request failed.', url)
+                continue
             parsed_tender = self.parse_ungm_notice(html, url, codes)
             if parsed_tender:
                 yield parsed_tender
@@ -247,11 +257,14 @@ def update_ungm_tenders(parsed_tenders):
     @staticmethod
     def download_document(tender_doc):
         with TemporaryFile() as content:
-            headers = {
-                'User-Agent': 'Mozilla/5.0'
-            }
             sleep(randint(2, 4))
-            response = requests.get(tender_doc.download_url, headers=headers, stream=True)
+            try:
+                response = requests.get(
+                    tender_doc.download_url, headers=GET_HEADERS,
+                    stream=True, timeout=REQUEST_TIMEOUT)
+            except requests.exceptions.RequestException as e:
+                logging.warning(e)
+                return
             if response.status_code == 200:
                 for chunk in response.iter_content(chunk_size=4096):
                     content.write(chunk)
diff --git a/app/server_requests.py b/app/server_requests.py
index bd5984b..141c0b0 100644
--- a/app/server_requests.py
+++ b/app/server_requests.py
@@ -7,9 +7,9 @@
 from random import randint
 from time import sleep
 
-from urllib.parse import urlparse
 from urllib3.exceptions import NewConnectionError
 
+from bs4 import BeautifulSoup
 from django.conf import settings
 
 from app.models import UNSPSCCode
@@ -39,6 +39,11 @@
         'NoticeTypes': [],
         'Reference': '',
         'DeadlineTo': '',
+        'IsSustainable': False,
+        'IsActive': True,
+        'NoticeDisplayType': None,
+        'NoticeSearchTotalLabelId': 'noticeSearchTotal',
+        'TypeOfCompetitions': [],
     },
     'awards': {
         'PageIndex': 0,
@@ -63,21 +68,35 @@
 }
 
 
+# Keep these consistent with a single current browser: UNGM sits behind a
+# bot filter and mismatched or outdated User-Agents get flagged.
+USER_AGENT = (
+    'Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0'
+)
+
+GET_HEADERS = {
+    'User-Agent': USER_AGENT,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+}
+
 POST_HEADERS = {
+    'User-Agent': USER_AGENT,
     'Accept': '*/*',
-    'Accept-Encoding': 'gzip, deflate',
-    'Accept-Language': 'en-US,en;q=0.5',
-    'Content-Type': 'application/json; charset=UTF-8',
-    'Host': urlparse(LIVE_ENDPOINT_URI).netloc,
-    'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:31.0)'
-    ' Gecko/20100101 Firefox/31.0',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Content-Type': 'application/json',
     'X-Requested-With': 'XMLHttpRequest',
+    'Origin': LIVE_ENDPOINT_URI,
 }
 
-GET_HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-    'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
-}
+# (connect, read) timeout in seconds for every request made to UNGM
+REQUEST_TIMEOUT = (10, 60)
+
+# Name of the ASP.NET antiforgery field rendered in the page; its value must
+# be sent back in the RequestVerificationToken header, together with the
+# matching cookie the server set on the same session.
+ANTIFORGERY_FIELD = '__RequestVerificationToken'
+ANTIFORGERY_HEADER = 'RequestVerificationToken'
 
 
 UNSPSC_CODES = UNSPSCCode.objects.values_list("id", flat=True)
@@ -96,12 +115,16 @@ class Requester(object):
 
     def get_request(self, url):
         try:
-            response = requests.get(url, headers=GET_HEADERS)
-        except requests.exceptions.ConnectionError:
+            response = requests.get(
+                url, headers=GET_HEADERS, timeout=REQUEST_TIMEOUT)
+        except requests.exceptions.RequestException as e:
+            logging.warning(e)
             return None
 
         if response.status_code == 200:
             return response.content
+        logging.warning(
+            'GET %s failed with status code: %s', url, response.status_code)
         return None
 
     def request_document(self, url):
@@ -141,47 +164,67 @@ def request(self, url, last_date, index):
                 url, url + '/Search', self.get_data(url, last_date, index))
             if html:
                 return html
-            sleep(randint(10, 15))
+            sleep(randint(30, 60))
         raise RequestsFailedError
 
-    def post_request(
-            self, get_url, post_url, data, headers=POST_HEADERS, content_type=None):
+    @staticmethod
+    def get_antiforgery_token(html):
+        soup = BeautifulSoup(html, 'html.parser')
+        field = soup.find('input', {'name': ANTIFORGERY_FIELD})
+        if field and field.get('value'):
+            return field['value']
+        meta = soup.find('meta', {'name': ANTIFORGERY_HEADER})
+        if meta and meta.get('content'):
+            return meta['content']
+        return None
+
+    def post_request(self, get_url, post_url, data, headers=None):
         """
-        AJAX-like POST request. Does a GET initially to receive cookies that
-        are used to the subsequent POST request.
+        AJAX-like POST request. Loads `get_url` first, in the same session,
+        to receive the cookies and the antiforgery token the POST requires.
 
         Returns HTML, NOT Response object.
         """
+        session = requests.Session()
+        session.cookies.set('UNGM.UserPreferredLanguage', 'en')
         try:
-            resp = requests.get(get_url, headers=GET_HEADERS)
-        except requests.exceptions.ConnectionError as e:
+            resp = session.get(
+                get_url, headers=GET_HEADERS, timeout=REQUEST_TIMEOUT)
+        except requests.exceptions.RequestException as e:
             logging.warning(e)
             return None
 
-        cookies = dict(resp.cookies)
-        cookies.update({'UNGM.UserPreferredLanguage': 'en'})
-        headers.update({
-            'Cookie': '; '.join(
-                ['{0}={1}'.format(k, v) for k, v in cookies.items()]),
+        if resp.status_code != 200:
+            logging.warning(
+                'GET %s failed with status code: %s', get_url,
+                resp.status_code)
+            return None
+
+        token = self.get_antiforgery_token(resp.content)
+        if not token:
+            logging.warning('No antiforgery token found on %s', get_url)
+            return None
+
+        post_headers = dict(POST_HEADERS, **(headers or {}))
+        post_headers.update({
+            ANTIFORGERY_HEADER: token,
             'Referer': get_url,
-            'Content-Length': str(len(data)),
         })
-        if content_type:
-            headers.update({'Content-Type': content_type})
 
         try:
             sleep(randint(2, 5))
-            resp = requests.post(post_url, data=data, cookies=cookies,
-                                 headers=headers)
-        except requests.exceptions.ConnectionError as e:
+            resp = session.post(
+                post_url, data=data, headers=post_headers,
+                timeout=REQUEST_TIMEOUT)
+        except requests.exceptions.RequestException as e:
             logging.warning(e)
             return None
 
-
         if resp.status_code == 200:
             return resp.content
 
-        logging.warning("Request failed with status code: " + str(resp.status_code))
+        logging.warning(
+            'POST %s failed with status code: %s', post_url, resp.status_code)
         return None
 
 #
diff --git a/app/tests/parser_files/ungm_notice_2026.html b/app/tests/parser_files/ungm_notice_2026.html
new file mode 100644
index 0000000..c28daae
--- /dev/null
+++ b/app/tests/parser_files/ungm_notice_2026.html
@@ -0,0 +1,155 @@
+<!DOCTYPE html>
+<html lang="en" dir="ltr">
+<head>
+    <meta charset="utf-8" />
+    <title>Procurement opportunities</title>
+</head>
+<body>
+    <input name="__RequestVerificationToken" type="hidden" value="test-token" />
+    <nav id="left" role="navigation">
+        <ul>
+            <li><a href="/">Home</a></li>
+            <li><a href="/Public/Notice">Procurement Opportunities</a></li>
+            <li>
+                <a href="/Public/UNGMPro">
+                    <span class="ungm-bold">UNGM Pro</span>
+                    <span class="status-tag sustainability">
+                        New!
+                    </span>
+                </a>
+            </li>
+        </ul>
+    </nav>
+    <main role="main" id="main-center">
+<div class="noticeDetail" id="noticeDetail">
+<div class="ungm-panel">
+    <div class="title">
+        <span class="highlighted">UNDP</span>
+    </div>
+    <div class="ungm-list-item ungm-background">
+        <div class="ungm-row">
+            <div class="ungm-col-lg-9 ungm-col-sm-12">
+                <span class="title">Fiscal space analysis 4 non-communicable diseases &amp; increased domestic resource</span>
+                <span class="status-tag"> Request for proposal</span>
+            </div>
+        </div>
+        <br />
+        <div class="ungm-col-sm-12">
+            <div class="row">
+                <span class="label">Reference:</span>
+                <span class="value">UNDP-SLE-00324</span>
+            </div>
+            <div class="row">
+                <span class="label">Beneficiary countries or territories:</span>
+                    <span class="value"> Sierra Leone</span>
+            </div>
+
+                <div class="row">
+                    <span class="label">Registration level:</span>
+                    <span class="value">Basic </span>
+                </div>
+                        <div class="row">
+                <span class="label">Published on:</span>
+                <span class="value">16-Sep-2026</span>
+            </div>
+            <div class="row">
+                <span class="label">Deadline on:</span>
+                <span class="value">07-Oct-2026 14:59 (GMT -4.00)</span>
+            </div>
+        </div>
+    </div>
+
+
+    <div class="ungm-list-item ungm-background">
+        <div class="title">Description</div>
+        <div> <div style="text-align:center"><font size="4"><b>RFP- 006/26 - </b></font><font face="Helvetica, Arial, sans-serif" size="4"><span style="letter-spacing:-0.133333px"><b>consultancy for a firm to provide Fiscal space analysis for non-communicable diseases and increased domestic resource mobilisation through health taxation</b></span></font></div></div>
+    </div>
+    <br />
+
+    <div class="accessibilityTabs">
+                    <div role="tabpanel"
+                 class="accessibilityTabpanel"
+                 id="contacts-tab"
+                 aria-labelledby="contacts">
+                <p>
+                    <div id="contactDetails">
+        <div class="ungm-list-item ungm-background">
+            <span class="title">
+
+            </span>
+
+            <div class="contactDetails">
+                    <div class="row">
+                        <span class="label"> Email address: </span>
+                        <span class="value"> <a href="mailto:vendors@example.org">vendors@example.org</a></span>
+                    </div>
+            </div>
+        </div>
+</div>
+                </p>
+            </div>
+                <div role="tabpanel"
+             class="accessibilityTabpanel"
+             id="unspscs-tab"
+             aria-labelledby="unspscs"
+             hidden="">
+            <p>
+<div class= "unspscSelector">
+    <div class= "unspscTree">
+    <div class="unspscNode"
+         data-isparent="True"
+         role=group>
+        <span class="nodeName
+            expanded
+            filtered"
+              tabindex="-1"
+              role="treeitem"
+              aria-expanded="true">
+            <span class="floatLeft">J</span>
+            <span class="floatLeft">&nbsp;-&nbsp;</span>
+            <span class="floatLeft">Services</span>
+        </span>
+                <div class="unspscChildren">
+    <div class="unspscNode"
+         data-isparent="True"
+         role=group>
+        <span class="nodeName
+            expanded
+            filtered"
+              tabindex="-1"
+              role="treeitem"
+              aria-expanded="true">
+            <span class="floatLeft">82000000</span>
+            <span class="floatLeft">&nbsp;-&nbsp;</span>
+            <span class="floatLeft">Editorial and Design and Graphic and Fine Art Services</span>
+        </span>
+                <div class="unspscChildren">
+    <div class="unspscNode"
+         data-isparent="False"
+         >
+        <span class="nodeName
+            "
+              tabindex="-1"
+              role="treeitem"
+              aria-expanded="true">
+            <span class="floatLeft">82111501</span>
+            <span class="floatLeft">&nbsp;-&nbsp;</span>
+            <span class="floatLeft">Instruction writing services</span>
+        </span>
+                <div class="unspscChildren">
+        </div>
+    </div>
+        </div>
+    </div>
+        </div>
+    </div>
+    </div>
+</div>
+            </p>
+        </div>
+    </div>
+</div>
+</div>
+    </main>
+</body>
+</html>
diff --git a/app/tests/parser_files/ungm_notice_list_2026.html b/app/tests/parser_files/ungm_notice_list_2026.html
new file mode 100644
index 0000000..416d09f
--- /dev/null
+++ b/app/tests/parser_files/ungm_notice_list_2026.html
@@ -0,0 +1,160 @@
+<div role="row" tabindex="0" data-noticeid="314599" class="tableRow dataRow notice-table">
+        <div role="cell" class="tableCell editable intendbuttonsHeader resultOptions">
+            <div class="ungm-flex-row ungm-flex-row--space-between ungm-flex-row--small-space-separator">
+                <span class='info-tooltip'>
+                    <a class="save-notice-button"
+                       data-action-url="/Vendor/MyUNGMPro/UnsaveNotice/314599"
+                       href="javascript:void(0);"
+                       style="display: none;">
+                    </a>
+
+                    <a class="save-notice-button" data-action-url="/Vendor/MyUNGMPro/SaveNotice/314599" href="javascript:void(0);" style="display: inline-flex !important;">
+                    </a>
+                    <span class='info-tooltip__text' style='white-space: normal; left: calc(-240px/2 + 45px/2);'>
+                        Subscribe to <a href='/Public/TenderAlertService' target='_blank'>UNGM Pro</a> to be able to save procurement opportunities.
+                    </span>
+                </span>
+
+                <div class="table-cell-eoi" style="display: block;">
+                    <input type="button"
+                           value='Express Interest'
+                           class=btnEOI buttonIntend
+                           data-noticeid="314599"
+                           data-agencyid="1"
+                           data-noticecanbeexpressedinterest="True"
+                           title="" />
+
+                </div>
+
+                <div class="table-cell-eoi" style="display: none;">
+                    <input type="button"
+                           value="View Documents"
+                           class="buttonViewDocuments" data-notice-id="314599" />
+                </div>
+
+            </div>
+        </div>
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+
+                <span class="ungm-title ungm-title--small">
+                    Fiscal space analysis 4 non-communicable diseases &amp; increased domestic resource
+                </span>
+            </div>
+
+            <a target='_blank' href='/Public/Notice/314599'>
+                    <svg class='open-in-new-tab__icon open-in-new-tab__icon--large' xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="black" width="18px" height="18px">
+                        <title>Open in a new window</title>
+                    </svg>
+            </a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultInfo1 deadline" data-description="Deadline">
+        <span>
+
+
+07-Oct-2026 14:59
+            (GMT -4.00)
+        </span>
+        <span class="remainingDaysToDeadline" style="display:none">0.108559707488426</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>
+
+16-Sep-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>UNDP</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span><label for='Request for proposal'>Request for proposal</label></span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>UNDP-SLE-00324</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Sierra Leone</span>
+    </div>
+
+
+
+</div>
+<script>
+    var deadlineCells = document.querySelectorAll('.tableCell.resultInfo1.deadline span');
+</script>
+
+
+<div role="row" tabindex="0" data-noticeid="316394" class="tableRow dataRow notice-table">
+        <div role="cell" class="tableCell editable intendbuttonsHeader resultOptions">
+            <div class="ungm-flex-row ungm-flex-row--space-between ungm-flex-row--small-space-separator">
+                <span class='info-tooltip'>
+                    <a class="save-notice-button" data-action-url="/Vendor/MyUNGMPro/SaveNotice/316394" href="javascript:void(0);" style="display: inline-flex !important;">
+                    </a>
+                </span>
+
+                <div class="table-cell-eoi" style="display: block;">
+                    <input type="button"
+                           value='Express Interest'
+                           class=btnEOI buttonIntend
+                           data-noticeid="316394"
+                           data-agencyid="128"
+                           data-noticecanbeexpressedinterest="True"
+                           title="" />
+
+                </div>
+            </div>
+        </div>
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+
+                <span class="ungm-title ungm-title--small">
+                    RFQ Adquisici&#243;n de sillas de ruedas Villavicencio
+                </span>
+            </div>
+
+            <a target='_blank' href='/Public/Notice/316394'>
+            </a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultInfo1 deadline" data-description="Deadline">
+        <span>
+
+
+07-Oct-2026 15:00
+            (GMT -4.00)
+        </span>
+        <span class="remainingDaysToDeadline" style="display:none">0.10925415193287</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>
+
+30-Sep-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>UN-Women</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span><label for='Request for quotation'>Request for quotation</label></span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>UNW-COL-2026-00071</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Colombia</span>
+    </div>
+
+
+
+</div>
+<script>
+    var deadlineCells = document.querySelectorAll('.tableCell.resultInfo1.deadline span');
+</script>
+
+
+<script type="text/javascript">
+    // set total value to label
+    var noticeTotal = "1576";
+    var noticeSearchTotalLabelId = "#noticeSearchTotal";
+</script>
diff --git a/app/tests/test_ungm_parser.py b/app/tests/test_ungm_parser.py
index 5116550..c8fe986 100644
--- a/app/tests/test_ungm_parser.py
+++ b/app/tests/test_ungm_parser.py
@@ -6,6 +6,7 @@
 
 from app.models import UNSPSCCode
 from app.parsers.ungm import UNGMWorker
+from app.server_requests import UNGMrequester
 from app.management.commands.add_award import Command
 from app.tests.base import BaseTestCase
 
@@ -56,6 +57,44 @@ def test_ungm_parse_notice_list_simple(self):
         self.assertEqual(tenders[1]['reference'], 'ILOAMM_NFQA_G20')
         self.assertEqual(tenders[1]['url'], expected_url)
 
+    def test_ungm_parse_notice_2026(self):
+        with open('app/tests/parser_files/ungm_notice_2026.html', 'r') as f:
+            html_string = f.read()
+
+        tender = self.worker.parse_ungm_notice(html_string, self.url,
+                                               self.unspsc_codes)
+
+        expected_deadline = make_aware(
+            datetime.strptime('07-Oct-2026 14:59', '%d-%b-%Y %H:%M'))
+        expected_deadline += timedelta(hours=4)
+        self.assertEqual(tender['tender']['reference'], 'UNDP-SLE-00324')
+        self.assertEqual(tender['tender']['notice_type'], 'Request for proposal')
+        self.assertEqual(tender['tender']['organization'], 'UNDP')
+        self.assertEqual(
+            tender['tender']['title'],
+            'Fiscal space analysis 4 non-communicable diseases & increased '
+            'domestic resource')
+        self.assertEqual(
+            tender['tender']['published'],
+            datetime.strptime('16-Sep-2026', '%d-%b-%Y').date())
+        self.assertEqual(tender['tender']['deadline'], expected_deadline)
+        self.assertIn('RFP- 006/26', tender['tender']['description'])
+        self.assertEqual(len(tender['documents']), 0)
+
+    def test_ungm_parse_notice_list_2026(self):
+        with open('app/tests/parser_files/ungm_notice_list_2026.html', 'r') as f:
+            html_string = f.read()
+
+        tenders = self.worker.parse_ungm_notice_list(html_string)
+        self.assertEqual(len(tenders), 2)
+        self.assertEqual(tenders[0]['reference'], 'UNDP-SLE-00324')
+        self.assertEqual(tenders[0]['published'], datetime.strptime('16-Sep-2026', '%d-%b-%Y').date())
+        self.assertEqual(
+            tenders[0]['url'], settings.UNGM_ENDPOINT_URI + '/Public/Notice/314599')
+        self.assertEqual(tenders[1]['reference'], 'UNW-COL-2026-00071')
+        self.assertEqual(
+            tenders[1]['url'], settings.UNGM_ENDPOINT_URI + '/Public/Notice/316394')
+
     def test_ungm_parse_award_simple(self):
         with open('app/tests/parser_files/base_award.html', 'r') as f:
             html_string = f.read()
@@ -116,3 +155,15 @@ def test_ungm_award_date_format(self):
 
         award = self.award.parse_award(html_string)
         self.assertEqual(award['award_date'], datetime.now().date())
+
+    def test_ungm_antiforgery_token(self):
+        html = (
+            '<form><input name="__RequestVerificationToken" type="hidden"'
+            ' value="token-value" /></form>'
+        )
+        self.assertEqual(
+            UNGMrequester.get_antiforgery_token(html), 'token-value')
+
+    def test_ungm_antiforgery_token_missing(self):
+        self.assertIsNone(
+            UNGMrequester.get_antiforgery_token('<html></html>'))

From 5b7f4835bf05590b62087f6fb3da84342fe40e33 Mon Sep 17 00:00:00 2001
From: andreighh <61791875+andreighh@users.noreply.github.com>
Date: Fri, 9 Oct 2026 15:51:02 +0300
Subject: [PATCH 2/3] Fix UNGM add_award after site changes

  - Search contract awards via /Public/ContractAward/PublicSearch with the
    payload the site sends now (SupplierCountries, no UNSPSCs)
  - Copy the awards payload instead of mutating the shared PAYLOAD dict
  - Extract the contract id in parse_contract_id, return None on no results
  - Fix save_award calling .save() on the update_or_create tuple
  - Store a missing contract value as None instead of '' (FloatField)
  - Tests and fixtures for the 2026 award search and award page- - Send UNSPSC ids as numbers and drop NoticeTASStatus, matching the site's search payload
---
 app/management/commands/add_award.py          |  32 ++--
 app/server_requests.py                        |  11 +-
 app/tests/parser_files/ungm_award_2026.html   | 161 ++++++++++++++++++
 .../parser_files/ungm_award_search_2026.html  |  77 +++++++++
 app/tests/test_ungm_parser.py                 |  20 ++-
 5 files changed, 281 insertions(+), 20 deletions(-)
 create mode 100644 app/tests/parser_files/ungm_award_2026.html
 create mode 100644 app/tests/parser_files/ungm_award_search_2026.html

diff --git a/app/management/commands/add_award.py b/app/management/commands/add_award.py
index 0a77007..97fabee 100644
--- a/app/management/commands/add_award.py
+++ b/app/management/commands/add_award.py
@@ -7,7 +7,9 @@
 from time import sleep
 from random import randint
 from app.models import Tender, Award, Vendor
-from app.server_requests import PAYLOAD, WINNERS_ENDPOINT_URI
+from app.server_requests import (
+    PAYLOAD, WINNERS_ENDPOINT_URI, WINNERS_SEARCH_URI
+)
 import logging
 from app.management.commands.base.params import BaseParamsUI
 
@@ -37,9 +39,8 @@ def handle(self, *args, **options):
             ).first()
 
             if not award:
-                try:
-                    contract_id = self.get_contract_id(tender.reference)
-                except TypeError:
+                contract_id = self.get_contract_id(tender.reference)
+                if not contract_id:
                     logger.warning(f'No award was found for the corresponding tender reference ({ tender.reference })')
                     continue
 
@@ -68,23 +69,27 @@ def get_contract_id(reference):
 
         requester = get_request_class(public=True)
 
-        payload = PAYLOAD['awards']
-        payload['Reference'] = reference
+        payload = dict(PAYLOAD['awards'], Reference=reference)
         for i in range(0, 3):
             resp = requester.post_request(
                 WINNERS_ENDPOINT_URI,
-                WINNERS_ENDPOINT_URI + '/Search',
+                WINNERS_SEARCH_URI,
                 json.dumps(payload),
             )
             if resp:
-                soup = BeautifulSoup(resp, 'html.parser')
-                contract_id = soup.find('div', {'class': 'tableRow dataRow'})['data-contractawardid']
-                return contract_id
-            sleep(randint(10, 15))
+                return Command.parse_contract_id(resp)
+            sleep(randint(30, 60))
 
         logger.error('POST request failed.')
         return None
 
+    @staticmethod
+    def parse_contract_id(html):
+        """ Return the id of the first contract award in a search result """
+        row = BeautifulSoup(html, 'html.parser').select_one(
+            'div.tableRow.dataRow[data-contractawardid]')
+        return row['data-contractawardid'] if row else None
+
     def parse_award(self, html):
         """ Parse a contract award HTML and return a dictionary with information
          such as: title, reference, vendor etc
@@ -105,7 +110,7 @@ def parse_award(self, html):
         award_fields = {
             'award_date': self.string_to_date(award_date) or datetime.date.today(),
             'vendors': vendor_list,
-            'value': float(value or 0) if value else '',
+            'value': float(value.strip().replace(',', '')) if value and value.strip() else None,
             'currency': '',
         }
 
@@ -122,8 +127,7 @@ def save_award(reference, award_fields):
         for vendor in vendors:
             vendor_object, _ = Vendor.objects.get_or_create(name=vendor)
             vendor_objects.append(vendor_object)
-        award = Award.objects.update_or_create(tender=tender_entry, **award_fields)
-        award.save()
+        award, _ = Award.objects.update_or_create(tender=tender_entry, **award_fields)
         award.vendors.add(*vendor_objects)
 
         return award
diff --git a/app/server_requests.py b/app/server_requests.py
index 141c0b0..4bec11f 100644
--- a/app/server_requests.py
+++ b/app/server_requests.py
@@ -17,6 +17,7 @@
 LIVE_ENDPOINT_URI = settings.UNGM_ENDPOINT_URI
 TENDERS_ENDPOINT_URI = LIVE_ENDPOINT_URI + '/Public/Notice'
 WINNERS_ENDPOINT_URI = LIVE_ENDPOINT_URI + '/Public/ContractAward'
+WINNERS_SEARCH_URI = WINNERS_ENDPOINT_URI + '/PublicSearch'
 SEARCH_UNSPSCS_URI = LIVE_ENDPOINT_URI + '/UNSPSC/Search'
 
 
@@ -24,7 +25,6 @@
     'tenders': {
         'PageIndex': 0,
         'PageSize': 15,
-        'NoticeTASStatus': [],
         'Description': '',
         'Title': '',
         'DeadlineFrom': '',
@@ -47,7 +47,7 @@
     },
     'awards': {
         'PageIndex': 0,
-        'PageSize': 100,
+        'PageSize': 15,
         'Title': '',
         'Description': '',
         'Reference': '',
@@ -55,8 +55,8 @@
         'AwardFrom': '',
         'AwardTo': '',
         'Countries': [],
+        'SupplierCountries': [],
         'Agencies': [],
-        'UNSPSCs': [],
         'SortField': 'AwardDate',
         'SortAscending': False,
     },
@@ -149,13 +149,14 @@ class UNGMrequester(Requester):
 
     def get_data(self, url, last_date, index):
         category = 'tenders' if 'Notice' in url else 'awards'
-        payload = PAYLOAD[category]
+        payload = dict(PAYLOAD[category])
         if category == 'tenders':
             today = datetime.now().strftime('%d-%b-%Y')
             payload['DeadlineFrom'] = payload['PublishedTo'] = today
             payload['PublishedFrom'] = last_date
             payload['PageIndex'] = index
-        payload['UNSPSCs'] = list(UNSPSC_CODES)
+            # UNGM expects the internal code ids as numbers
+            payload['UNSPSCs'] = [int(code) for code in UNSPSC_CODES]
         return json.dumps(payload)
 
     def request(self, url, last_date, index):
diff --git a/app/tests/parser_files/ungm_award_2026.html b/app/tests/parser_files/ungm_award_2026.html
new file mode 100644
index 0000000..f23836e
--- /dev/null
+++ b/app/tests/parser_files/ungm_award_2026.html
@@ -0,0 +1,161 @@
+<div class="awardDetail">
+
+    <script>
+        $(document).ready(function () {
+            UNGM.ExpandableContacts.init();
+            UNGM.UNSPSC.init();
+            UNGM.Tab.init();
+        });
+    </script>
+
+    <div class="accessibilityTabs" >
+        <div role="tablist" class="accessibilityTablist" aria-label="Contract award detail">
+            <button role="tab"
+                    class="accessibilityTab"
+                    aria-selected="true"
+                    aria-controls="General-tab"
+                    id="General"
+                    tabindex="0">
+            General</button>
+            <button role="tab"
+                    class="accessibilityTab"
+                    aria-selected="false"
+                    aria-controls="Vendors-tab"
+                    id="Vendors"
+                    tabindex="-1">
+            Supplier</button>
+            <button role="tab"
+                    class="accessibilityTab"
+                    aria-selected="false"
+                    aria-controls="Contacts-tab"
+                    id="Contacts"
+                    tabindex="-1">
+            Contacts</button>
+            <button role="tab"
+                    class="accessibilityTab"
+                    aria-selected="false"
+                    aria-controls="UNSPSC-tab"
+                    id="UNSPSC"
+                    tabindex="-1">
+            UNSPSC codes</button>
+        </div>
+
+        <div role="tabpanel"
+             class="accessibilityTabpanel"
+             id="General-tab"
+             aria-labelledby="General">
+            <p>
+                <fieldset>
+    <legend>General information</legend>
+    <div class="reportRow">
+        <label for="Title">Title</label>
+        <span id="awardTitle" class="value">SUPPLY AND INSTALLATION OF DRIP IRRIGATION SYSTEM FOR 5 HECTARES AT CHAMINUKA VOCATIONAL TRAINING CENTRE</span>
+    </div>
+    <div class="reportRow">
+        <label for="AgencyId">Organization</label>
+        <span class="value">International Labour Organization</span>
+    </div>
+    <div class="reportRow">
+        <label for="Reference">Reference</label>
+        <span class="value">rfx_10996_ROAF</span>
+    </div>
+    <div class="reportRow">
+        <label for="AwardDate">Award date</label>
+        <span class="value">
+
+09-Oct-2026</span>
+    </div>
+    <div class="reportRow">
+        <label for="ContractValue">Contract value (USD)</label>
+
+        <span class="value"></span>
+    </div>
+    <div class="reportRow">
+        <label for="Description">Description</label>
+    </div>
+    <div class="raw clear">
+        Annex 0 - Invitation to Bidders
+    </div>
+</fieldset>
+
+
+                <fieldset>
+    <legend>Countries or territories</legend>
+        <div class="formRow clear">
+            Zimbabwe
+        </div>
+</fieldset>
+
+
+            </p>
+        </div>
+        <div role="tabpanel"
+             class="accessibilityTabpanel"
+             id="Vendors-tab"
+             aria-labelledby="Vendors"
+             hidden="">
+            <p>
+
+
+<div id="contractAwardVendorsContainer">
+
+
+
+        <div class="editableListItem">
+            <label>
+                Nyanzou Canvas Works
+            </label>
+        </div>
+
+</div>
+
+            </p>
+        </div>
+        <div role="tabpanel"
+             class="accessibilityTabpanel"
+             id="Contacts-tab"
+             aria-labelledby="Contacts"
+             hidden="">
+            <p>
+                <input type="button" class="btnCollapseAll" value="Collapse all" style="float:right;" />
+<input type="button" class="btnExpandAll" value="Expand all" style="float:right;" />
+<div class="clear"></div>
+
+    <div class="editableListItem">
+        <span class="options">
+            <input type="button" value="Show details" />
+        </span>
+
+
+        <span>
+            <strong>
+
+            </strong>
+        </span>
+        <div class="clear"></div>
+
+        <div class="contactDetails" style="display: none;">
+            <div class="reportRow">
+                <label for="FirstName">First name</label>
+                <span class="value">PROCUREMENT</span>
+            </div>
+            <div class="reportRow">
+                <label for="Email">Contact email</label>
+                <span class="value">pcrt@ilo.org</span>
+            </div>
+        </div>
+    </div>
+
+            </p>
+        </div>
+        <div role="tabpanel"
+             class="accessibilityTabpanel"
+             id="UNSPSC-tab"
+             aria-labelledby="UNSPSC"
+             hidden="">
+            <p>
+            </p>
+
+        </div>
+    </div>
+</div>
diff --git a/app/tests/parser_files/ungm_award_search_2026.html b/app/tests/parser_files/ungm_award_search_2026.html
new file mode 100644
index 0000000..1728173
--- /dev/null
+++ b/app/tests/parser_files/ungm_award_search_2026.html
@@ -0,0 +1,77 @@
+<div role="row" tabindex="0" data-contractawardid="160222" class="tableRow dataRow">
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+                <span class="ungm-title ungm-title--small">
+                    SUPPLY AND INSTALLATION OF DRIP IRRIGATION SYSTEM FOR 5 HECTARES AT CHAMINUKA VOCATIONAL TRAINING CENTRE
+                </span>
+            </div>
+            <a target='_blank' href='/Public/ContractAward/160222'>
+
+                    <svg class='open-in-new-tab__icon open-in-new-tab__icon--large' xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="black" width="18px" height="18px">
+                        <title>Open in a new window</title>
+                        <path d="M0 0h24v24H0z" fill="none"/><path d="M19 19H5V5h7V3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2v-7h-2v7zM14 3v2h3.59l-9.83 9.83 1.41 1.41L19 6.41V10h2V3h-7z"/>
+                    </svg>
+
+            </a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>Nyanzou Canvas Works</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Award date">
+        <span>
+
+09-Oct-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>ILO</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>rfx_10996_ROAF</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Zimbabwe</span>
+    </div>
+</div>
+<div role="row" tabindex="0" data-contractawardid="160161" class="tableRow dataRow">
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+                <span class="ungm-title ungm-title--small">
+                    Servicio de inscripci&#243;n para la participaci&#243;n de funcionarios del INADEH en XCOM LATAM 2026
+                </span>
+            </div>
+            <a target='_blank' href='/Public/ContractAward/160161'>
+
+                    <svg class='open-in-new-tab__icon open-in-new-tab__icon--large' xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="black" width="18px" height="18px">
+                        <title>Open in a new window</title>
+                        <path d="M0 0h24v24H0z" fill="none"/><path d="M19 19H5V5h7V3H5c-1.11 0-2 .9-2 2v14c0 1.1.89 2 2 2h14c1.1 0 2-.9 2-2v-7h-2v7zM14 3v2h3.59l-9.83 9.83 1.41 1.41L19 6.41V10h2V3h-7z"/>
+                    </svg>
+
+            </a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>2 M PUBLICACIONES, S.A.</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Award date">
+        <span>
+
+08-Oct-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>ILO</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>rfx_9732_ROLAC</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Panama</span>
+    </div>
+</div>
+
+<script type="text/javascript">
+    //  set total value to label
+    $("#contractAwardsTotal").find("#searchTotal").html(49473);
+ </script>
diff --git a/app/tests/test_ungm_parser.py b/app/tests/test_ungm_parser.py
index c8fe986..130af2c 100644
--- a/app/tests/test_ungm_parser.py
+++ b/app/tests/test_ungm_parser.py
@@ -145,7 +145,7 @@ def test_ungm_award_all_empty(self):
         award = self.award.parse_award(html_string)
 
         self.assertEqual(award['vendors'][0], '')
-        self.assertEqual(award['value'], '')
+        self.assertIsNone(award['value'])
         self.assertEqual(award['currency'], '')
         self.assertEqual(award['award_date'], datetime.now().date())
 
@@ -156,6 +156,24 @@ def test_ungm_award_date_format(self):
         award = self.award.parse_award(html_string)
         self.assertEqual(award['award_date'], datetime.now().date())
 
+    def test_ungm_parse_award_2026(self):
+        with open('app/tests/parser_files/ungm_award_2026.html', 'r') as f:
+            html_string = f.read()
+
+        award = self.award.parse_award(html_string)
+        self.assertEqual(award['vendors'], ['Nyanzou Canvas Works'])
+        self.assertEqual(
+            award['award_date'], datetime.strptime('09-Oct-2026', '%d-%b-%Y'))
+        self.assertIsNone(award['value'])
+        self.assertEqual(award['currency'], '')
+
+    def test_ungm_award_search_contract_id(self):
+        with open('app/tests/parser_files/ungm_award_search_2026.html', 'r') as f:
+            html_string = f.read()
+
+        self.assertEqual(self.award.parse_contract_id(html_string), '160222')
+        self.assertIsNone(self.award.parse_contract_id('<div></div>'))
+
     def test_ungm_antiforgery_token(self):
         html = (
             '<form><input name="__RequestVerificationToken" type="hidden"'

From f2ac822f213a0ae076135d2706053c94c1298b32 Mon Sep 17 00:00:00 2001
From: andreighh <61791875+andreighh@users.noreply.github.com>
Date: Fri, 9 Oct 2026 19:52:51 +0300
Subject: [PATCH 3/3] Handle multiple UNGM contract awards per tender

  - Only search awards of tenders that expired in the last 90 days, and
    pause between requests, to keep the request volume to UNGM low
  - Match search results on the exact tender reference (UNGM's filter
    also matches partially)
  - Fetch every matching award (one per lot) and merge them into one
    Award like TED and IUCN: summed value, all vendors, latest date
  - Normalize vendor names with transform_vendor_name
  - Return dates from string_to_date so award dates can be compared
  - Describe what the command does in its help text
---
 app/management/commands/add_award.py          | 114 ++++++++++++------
 .../parser_files/ungm_award_search_lots.html  |  87 +++++++++++++
 app/tests/test_ungm_parser.py                 |  50 +++++++-
 3 files changed, 211 insertions(+), 40 deletions(-)
 create mode 100644 app/tests/parser_files/ungm_award_search_lots.html

diff --git a/app/management/commands/add_award.py b/app/management/commands/add_award.py
index 97fabee..0315ee1 100644
--- a/app/management/commands/add_award.py
+++ b/app/management/commands/add_award.py
@@ -7,6 +7,7 @@
 from time import sleep
 from random import randint
 from app.models import Tender, Award, Vendor
+from app.utils import transform_vendor_name
 from app.server_requests import (
     PAYLOAD, WINNERS_ENDPOINT_URI, WINNERS_SEARCH_URI
 )
@@ -23,36 +24,50 @@
 CSS_VALUE = 'ContractValue'
 CSS_VENDOR_LIST = 'contractAwardVendorsContainer'
 
+# Only look for awards of tenders whose deadline passed within this many days.
+# Each tender costs at least 3 requests to UNGM, so older tenders that never got an
+# award published would otherwise be searched again on every run.
+AWARD_SEARCH_DAYS = 90
+
 
 class Command(BaseCommand, BaseParamsUI):
-    help = 'Gets all awards from the past day'
+    help = (
+        'Finds the contract awards of UNGM tenders whose deadline passed in '
+        f'the last {AWARD_SEARCH_DAYS} days and have no award yet. TED and '
+        'IUCN awards are imported by update_ted and update_iucn.'
+    )
 
     def handle(self, *args, **options):
+        now = datetime.datetime.now(timezone.utc)
         expired_tenders = Tender.objects.filter(
-            deadline__lt=datetime.datetime.now(timezone.utc),
-            source='UNGM'
+            source='UNGM',
+            deadline__lt=now,
+            deadline__gte=now - datetime.timedelta(days=AWARD_SEARCH_DAYS),
+            awards__isnull=True,
         )
 
+        requester = get_request_class(public=True)
         for tender in expired_tenders:
-            award = Award.objects.filter(
-                tender__reference=tender.reference
-            ).first()
-
-            if not award:
-                contract_id = self.get_contract_id(tender.reference)
-                if not contract_id:
-                    logger.warning(f'No award was found for the corresponding tender reference ({ tender.reference })')
-                    continue
-
-                request_cls = get_request_class(public=True)
-                url = '/'.join((WINNERS_ENDPOINT_URI, contract_id))
-                html_data = request_cls.get_request(url)
-
-                try:
-                    award_fields = self.parse_award(html_data)
-                    self.save_award(tender.reference, award_fields)
-                except TypeError:
-                    logger.error('Contract does not exist!')
+            # Space out the requests to UNGM, like a person browsing
+            sleep(randint(2, 5))
+            contract_ids = self.get_contract_ids(tender.reference)
+            if not contract_ids:
+                logger.warning(f'No award was found for the corresponding tender reference ({ tender.reference })')
+                continue
+
+            # A tender split into lots has one contract award per lot
+            awards = []
+            for contract_id in contract_ids:
+                sleep(randint(2, 5))
+                html_data = requester.get_request(
+                    '/'.join((WINNERS_ENDPOINT_URI, contract_id)))
+                if html_data:
+                    awards.append(self.parse_award(html_data))
+                else:
+                    logger.error(f'Could not load contract award {contract_id}')
+
+            if awards:
+                self.save_award(tender.reference, self.merge_awards(awards))
 
     @staticmethod
     def find_by_label(soup, label):
@@ -62,10 +77,10 @@ def find_by_label(soup, label):
             return ''
 
     @staticmethod
-    def get_contract_id(reference):
+    def get_contract_ids(reference):
         if len(reference) < 3:
             logger.error('The search text must be at least 3 characters long.')
-            return
+            return []
 
         requester = get_request_class(public=True)
 
@@ -77,18 +92,46 @@ def get_contract_id(reference):
                 json.dumps(payload),
             )
             if resp:
-                return Command.parse_contract_id(resp)
+                return Command.parse_contract_ids(resp, reference)
             sleep(randint(30, 60))
 
         logger.error('POST request failed.')
-        return None
+        return []
 
     @staticmethod
-    def parse_contract_id(html):
-        """ Return the id of the first contract award in a search result """
-        row = BeautifulSoup(html, 'html.parser').select_one(
-            'div.tableRow.dataRow[data-contractawardid]')
-        return row['data-contractawardid'] if row else None
+    def parse_contract_ids(html, reference):
+        """
+        Return the ids of the contract awards in a search result whose
+        reference is exactly `reference`. UNGM's reference filter also
+        matches partially (e.g. "RFQ/2026/640" finds "RFQ/2026/64033").
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        contract_ids = []
+        for row in soup.select('div.tableRow.dataRow[data-contractawardid]'):
+            cell = row.find('div', attrs={'data-description': 'Reference'})
+            if cell and cell.get_text().strip() == reference.strip():
+                contract_ids.append(row['data-contractawardid'])
+        return contract_ids
+
+    @staticmethod
+    def merge_awards(awards):
+        """
+        Merge the contract awards of one tender (one per lot) into a single
+        award, like the TED and IUCN parsers do: values are added up, vendors
+        collected and the latest award date kept.
+        """
+        values = [award['value'] for award in awards if award['value'] is not None]
+        vendors = []
+        for award in awards:
+            for vendor in award['vendors']:
+                if vendor and vendor not in vendors:
+                    vendors.append(vendor)
+        return {
+            'award_date': max(award['award_date'] for award in awards),
+            'vendors': vendors,
+            'value': sum(values) if values else None,
+            'currency': 'USD' if values else '',
+        }
 
     def parse_award(self, html):
         """ Parse a contract award HTML and return a dictionary with information
@@ -125,9 +168,11 @@ def save_award(reference, award_fields):
         vendors = award_fields.pop('vendors')
         vendor_objects = []
         for vendor in vendors:
-            vendor_object, _ = Vendor.objects.get_or_create(name=vendor)
+            vendor_object, _ = Vendor.objects.get_or_create(
+                name=transform_vendor_name(vendor))
             vendor_objects.append(vendor_object)
-        award, _ = Award.objects.update_or_create(tender=tender_entry, **award_fields)
+        award, _ = Award.objects.update_or_create(
+            tender=tender_entry, defaults=award_fields)
         award.vendors.add(*vendor_objects)
 
         return award
@@ -142,5 +187,6 @@ def to_unicode(string):
     @staticmethod
     def string_to_date(string_date):
         if string_date:
-            return datetime.datetime.strptime(string_date.strip(), '%d-%b-%Y')
+            return datetime.datetime.strptime(
+                string_date.strip(), '%d-%b-%Y').date()
         return None
diff --git a/app/tests/parser_files/ungm_award_search_lots.html b/app/tests/parser_files/ungm_award_search_lots.html
new file mode 100644
index 0000000..e40a594
--- /dev/null
+++ b/app/tests/parser_files/ungm_award_search_lots.html
@@ -0,0 +1,87 @@
+<div role="row" tabindex="0" data-contractawardid="501" class="tableRow dataRow">
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+                <span class="ungm-title ungm-title--small">
+                    Software licences - Lot 1
+                </span>
+            </div>
+            <a target='_blank' href='/Public/ContractAward/501'></a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>Supplier One Ltd</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Award date">
+        <span>
+
+08-Oct-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>UNOPS</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>RFQ/2026/640</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Somalia</span>
+    </div>
+</div>
+<div role="row" tabindex="0" data-contractawardid="502" class="tableRow dataRow">
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+                <span class="ungm-title ungm-title--small">
+                    Another tender whose reference only starts the same
+                </span>
+            </div>
+            <a target='_blank' href='/Public/ContractAward/502'></a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>Unrelated Supplier</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Award date">
+        <span>
+
+08-Oct-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>UNOPS</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>RFQ/2026/64033</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Sierra Leone</span>
+    </div>
+</div>
+<div role="row" tabindex="0" data-contractawardid="503" class="tableRow dataRow">
+    <div role="cell" class="tableCell resultTitle">
+        <div class="ungm-flex-row ungm-flex-row--space-between">
+            <div class="ungm-flex-row">
+                <span class="ungm-title ungm-title--small">
+                    Software licences - Lot 2
+                </span>
+            </div>
+            <a target='_blank' href='/Public/ContractAward/503'></a>
+        </div>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>Supplier Two Ltd</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Award date">
+        <span>
+
+07-Oct-2026</span>
+    </div>
+    <div role="cell" class="tableCell resultAgency">
+        <span>UNOPS</span>
+    </div>
+    <div role="cell" class="tableCell resultInfo1" data-description="Reference">
+        <span>RFQ/2026/640</span>
+    </div>
+    <div role="cell" class="tableCell">
+        <span>Somalia</span>
+    </div>
+</div>
diff --git a/app/tests/test_ungm_parser.py b/app/tests/test_ungm_parser.py
index 130af2c..096e577 100644
--- a/app/tests/test_ungm_parser.py
+++ b/app/tests/test_ungm_parser.py
@@ -100,7 +100,7 @@ def test_ungm_parse_award_simple(self):
             html_string = f.read()
 
         award = self.award.parse_award(html_string)
-        expected_date = datetime.strptime('18-Sep-2019', '%d-%b-%Y')
+        expected_date = datetime.strptime('18-Sep-2019', '%d-%b-%Y').date()
         self.assertEqual(award['vendors'][0], 'E-Secure Sàrl')
         self.assertEqual(award['value'], 25000.00)
         self.assertEqual(award['currency'], 'USD')
@@ -162,17 +162,55 @@ def test_ungm_parse_award_2026(self):
 
         award = self.award.parse_award(html_string)
         self.assertEqual(award['vendors'], ['Nyanzou Canvas Works'])
-        self.assertEqual(
-            award['award_date'], datetime.strptime('09-Oct-2026', '%d-%b-%Y'))
+        self.assertEqual(award['award_date'], date(2026, 10, 9))
         self.assertIsNone(award['value'])
         self.assertEqual(award['currency'], '')
 
-    def test_ungm_award_search_contract_id(self):
+    def test_ungm_award_search_contract_ids(self):
         with open('app/tests/parser_files/ungm_award_search_2026.html', 'r') as f:
             html_string = f.read()
 
-        self.assertEqual(self.award.parse_contract_id(html_string), '160222')
-        self.assertIsNone(self.award.parse_contract_id('<div></div>'))
+        self.assertEqual(
+            self.award.parse_contract_ids(html_string, 'rfx_10996_ROAF'),
+            ['160222'])
+        self.assertEqual(
+            self.award.parse_contract_ids(html_string, 'rfx_10996'), [])
+        self.assertEqual(
+            self.award.parse_contract_ids('<div></div>', 'rfx_10996_ROAF'), [])
+
+    def test_ungm_award_search_lots(self):
+        # Two lots of RFQ/2026/640, and RFQ/2026/64033, which UNGM also
+        # returns because its reference filter matches partially
+        with open('app/tests/parser_files/ungm_award_search_lots.html', 'r') as f:
+            html_string = f.read()
+
+        self.assertEqual(
+            self.award.parse_contract_ids(html_string, 'RFQ/2026/640'),
+            ['501', '503'])
+
+    def test_ungm_merge_awards(self):
+        award = self.award.merge_awards([
+            {'award_date': date(2026, 10, 7), 'vendors': ['Supplier One'],
+             'value': 1000.0, 'currency': 'USD'},
+            {'award_date': date(2026, 10, 8), 'vendors': ['Supplier Two', ''],
+             'value': None, 'currency': ''},
+            {'award_date': date(2026, 10, 6), 'vendors': ['Supplier One'],
+             'value': 500.0, 'currency': 'USD'},
+        ])
+
+        self.assertEqual(award['award_date'], date(2026, 10, 8))
+        self.assertEqual(award['vendors'], ['Supplier One', 'Supplier Two'])
+        self.assertEqual(award['value'], 1500.0)
+        self.assertEqual(award['currency'], 'USD')
+
+    def test_ungm_merge_awards_without_values(self):
+        award = self.award.merge_awards([
+            {'award_date': date(2026, 10, 9), 'vendors': ['Supplier'],
+             'value': None, 'currency': ''},
+        ])
+
+        self.assertIsNone(award['value'])
+        self.assertEqual(award['currency'], '')
 
     def test_ungm_antiforgery_token(self):
         html = (