Fix UNGM scraper/parser

app/management/commands/add_award.py

@@ -7,7 +7,10 @@
 from time import sleep
 from random import randint
 from app.models import Tender, Award, Vendor
-from app.server_requests import PAYLOAD, WINNERS_ENDPOINT_URI
+from app.utils import transform_vendor_name
+from app.server_requests import (
+    PAYLOAD, WINNERS_ENDPOINT_URI, WINNERS_SEARCH_URI
+)
 import logging
 from app.management.commands.base.params import BaseParamsUI
 
@@ -21,37 +24,50 @@
 CSS_VALUE = 'ContractValue'
 CSS_VENDOR_LIST = 'contractAwardVendorsContainer'
 
+# Only look for awards of tenders whose deadline passed within this many days.
+# Each tender costs at least 3 requests to UNGM, so older tenders that never got an
+# award published would otherwise be searched again on every run.
+AWARD_SEARCH_DAYS = 90
+
 
 class Command(BaseCommand, BaseParamsUI):
-    help = 'Gets all awards from the past day'
+    help = (
+        'Finds the contract awards of UNGM tenders whose deadline passed in '
+        f'the last {AWARD_SEARCH_DAYS} days and have no award yet. TED and '
+        'IUCN awards are imported by update_ted and update_iucn.'
+    )
 
     def handle(self, *args, **options):
+        now = datetime.datetime.now(timezone.utc)
         expired_tenders = Tender.objects.filter(
-            deadline__lt=datetime.datetime.now(timezone.utc),
-            source='UNGM'
+            source='UNGM',
+            deadline__lt=now,
+            deadline__gte=now - datetime.timedelta(days=AWARD_SEARCH_DAYS),
+            awards__isnull=True,
         )
 
+        requester = get_request_class(public=True)
         for tender in expired_tenders:
-            award = Award.objects.filter(
-                tender__reference=tender.reference
-            ).first()
-
-            if not award:
-                try:
-                    contract_id = self.get_contract_id(tender.reference)
-                except TypeError:
-                    logger.warning(f'No award was found for the corresponding tender reference ({ tender.reference })')
-                    continue
-
-                request_cls = get_request_class(public=True)
-                url = '/'.join((WINNERS_ENDPOINT_URI, contract_id))
-                html_data = request_cls.get_request(url)
-
-                try:
-                    award_fields = self.parse_award(html_data)
-                    self.save_award(tender.reference, award_fields)
-                except TypeError:
-                    logger.error('Contract does not exist!')
+            # Space out the requests to UNGM, like a person browsing
+            sleep(randint(2, 5))
+            contract_ids = self.get_contract_ids(tender.reference)
+            if not contract_ids:
+                logger.warning(f'No award was found for the corresponding tender reference ({ tender.reference })')
+                continue
+
+            # A tender split into lots has one contract award per lot
+            awards = []
+            for contract_id in contract_ids:
+                sleep(randint(2, 5))
+                html_data = requester.get_request(
+                    '/'.join((WINNERS_ENDPOINT_URI, contract_id)))
+                if html_data:
+                    awards.append(self.parse_award(html_data))
+                else:
+                    logger.error(f'Could not load contract award {contract_id}')
+
+            if awards:
+                self.save_award(tender.reference, self.merge_awards(awards))
 
     @staticmethod
     def find_by_label(soup, label):
@@ -61,29 +77,61 @@ def find_by_label(soup, label):
             return ''
 
     @staticmethod
-    def get_contract_id(reference):
+    def get_contract_ids(reference):
         if len(reference) < 3:
             logger.error('The search text must be at least 3 characters long.')
-            return
+            return []
 
         requester = get_request_class(public=True)
 
-        payload = PAYLOAD['awards']
-        payload['Reference'] = reference
+        payload = dict(PAYLOAD['awards'], Reference=reference)
         for i in range(0, 3):
             resp = requester.post_request(
                 WINNERS_ENDPOINT_URI,
-                WINNERS_ENDPOINT_URI + '/Search',
+                WINNERS_SEARCH_URI,
                 json.dumps(payload),
             )
             if resp:
-                soup = BeautifulSoup(resp, 'html.parser')
-                contract_id = soup.find('div', {'class': 'tableRow dataRow'})['data-contractawardid']
-                return contract_id
-            sleep(randint(10, 15))
+                return Command.parse_contract_ids(resp, reference)
+            sleep(randint(30, 60))
 
         logger.error('POST request failed.')
-        return None
+        return []
+
+    @staticmethod
+    def parse_contract_ids(html, reference):
+        """
+        Return the ids of the contract awards in a search result whose
+        reference is exactly `reference`. UNGM's reference filter also
+        matches partially (e.g. "RFQ/2026/640" finds "RFQ/2026/64033").
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        contract_ids = []
+        for row in soup.select('div.tableRow.dataRow[data-contractawardid]'):
+            cell = row.find('div', attrs={'data-description': 'Reference'})
+            if cell and cell.get_text().strip() == reference.strip():
+                contract_ids.append(row['data-contractawardid'])
+        return contract_ids
+
+    @staticmethod
+    def merge_awards(awards):
+        """
+        Merge the contract awards of one tender (one per lot) into a single
+        award, like the TED and IUCN parsers do: values are added up, vendors
+        collected and the latest award date kept.
+        """
+        values = [award['value'] for award in awards if award['value'] is not None]
+        vendors = []
+        for award in awards:
+            for vendor in award['vendors']:
+                if vendor and vendor not in vendors:
+                    vendors.append(vendor)
+        return {
+            'award_date': max(award['award_date'] for award in awards),
+            'vendors': vendors,
+            'value': sum(values) if values else None,
+            'currency': 'USD' if values else '',
+        }
 
     def parse_award(self, html):
         """ Parse a contract award HTML and return a dictionary with information
@@ -105,7 +153,7 @@ def parse_award(self, html):
         award_fields = {
             'award_date': self.string_to_date(award_date) or datetime.date.today(),
             'vendors': vendor_list,
-            'value': float(value or 0) if value else '',
+            'value': float(value.strip().replace(',', '')) if value and value.strip() else None,
             'currency': '',
         }
 
@@ -120,10 +168,11 @@ def save_award(reference, award_fields):
         vendors = award_fields.pop('vendors')
         vendor_objects = []
         for vendor in vendors:
-            vendor_object, _ = Vendor.objects.get_or_create(name=vendor)
+            vendor_object, _ = Vendor.objects.get_or_create(
+                name=transform_vendor_name(vendor))
             vendor_objects.append(vendor_object)
-        award = Award.objects.update_or_create(tender=tender_entry, **award_fields)
-        award.save()
+        award, _ = Award.objects.update_or_create(
+            tender=tender_entry, defaults=award_fields)
         award.vendors.add(*vendor_objects)
 
         return award
@@ -138,5 +187,6 @@ def to_unicode(string):
     @staticmethod
     def string_to_date(string_date):
         if string_date:
-            return datetime.datetime.strptime(string_date.strip(), '%d-%b-%Y')
+            return datetime.datetime.strptime(
+                string_date.strip(), '%d-%b-%Y').date()
         return None

app/parsers/ungm.py

@@ -14,7 +14,9 @@
 
 from app.exceptions import UNSPCCodesNotFound
 from app.models import UNSPSCCode, WorkerLog, Tender, TenderDocument
-from app.server_requests import get_request_class
+from app.server_requests import (
+    get_request_class, GET_HEADERS, REQUEST_TIMEOUT
+)
 from scratch import settings
 
 
@@ -65,14 +67,15 @@ def parse_ungm_notice_list(html):
 
         tenders_list = []
         for tender in tenders:
+            # Columns: options, title, deadline, published, organization,
+            # type, reference, country
+            cells = tender.find_all('div', class_='tableCell', recursive=False)
+            href = cells[1].a['href'].strip() if cells[1].a else ''
             tender_dict = {
                 'published': UNGMWorker.parse_date(
-                    tender.contents[7].span.string.strip(), '%d-%b-%Y'),
-                'reference': tender.contents[13].span.string.strip(),
-                'url': (
-                    endpoint + tender.contents[3].a['href'].strip()
-                    if tender.contents[3].a['href'].strip() else ''
-                ),
+                    cells[3].span.get_text().strip(), '%d-%b-%Y'),
+                'reference': cells[6].span.get_text().strip(),
+                'url': endpoint + href if href else '',
             }
             if tender_dict['url'] == 'http://google.com':
                 # This is a bug that should be investigated
@@ -96,6 +99,9 @@ def parse_ungm_notice(html, url, codes):
             the tender is invalid.
         """
         soup = BeautifulSoup(html, 'html.parser')
+        # Restrict to the notice itself, the page menu also uses classes
+        # like "status-tag"
+        soup = soup.find(id='noticeDetail') or soup
         documents = UNGMWorker.find_by_class(soup, "lnkShowDocument", "a")
         description = UNGMWorker.find_by_class(
             soup, "ungm-list-item ungm-background", "div")
@@ -175,7 +181,11 @@ def parsed_tenders(self, tenders):
                 url = tender.url
             except AttributeError:
                 url = tender['url']
+            sleep(randint(2, 5))
             html = self.requester.get_request(url)
+            if not html:
+                logging.warning('Skipping UNGM tender %s, request failed.', url)
+                continue
             parsed_tender = self.parse_ungm_notice(html, url, codes)
             if parsed_tender:
                 yield parsed_tender
@@ -247,11 +257,14 @@ def update_ungm_tenders(parsed_tenders):
     @staticmethod
     def download_document(tender_doc):
         with TemporaryFile() as content:
-            headers = {
-                'User-Agent': 'Mozilla/5.0'
-            }
             sleep(randint(2, 4))
-            response = requests.get(tender_doc.download_url, headers=headers, stream=True)
+            try:
+                response = requests.get(
+                    tender_doc.download_url, headers=GET_HEADERS,
+                    stream=True, timeout=REQUEST_TIMEOUT)
+            except requests.exceptions.RequestException as e:
+                logging.warning(e)
+                return
             if response.status_code == 200:
                 for chunk in response.iter_content(chunk_size=4096):
                     content.write(chunk)