Merge pull request #112 from dev195/master

Switch to geoip2

Author: dev195

README.md

@@ -14,8 +14,8 @@ Key features:
 
 * Linux (developed on Ubuntu 12.04)
 * Python 2.7
-* [pygeoip](https://github.com/appliedsec/pygeoip), GNU Lesser GPL
-  * [MaxMind GeoIP Legacy datasets](http://dev.maxmind.com/geoip/legacy/geolite/)
+* [geoip2](https://github.com/maxmind/GeoIP2-python), Apache License, Version 2.0
+  * [MaxMind GeoIP datasets](https://dev.maxmind.com/geoip/geoip2/geolite2/)
 * [PyCrypto](https://pypi.python.org/pypi/pycrypto), custom license
 * [dpkt](https://code.google.com/p/dpkt/), New BSD License
 * [IPy](https://github.com/haypo/python-ipy), BSD 2-Clause License
@@ -24,13 +24,11 @@ Key features:
 
 ## Installation
 
-1. Install all of the necessary Python modules listed above. Many of them are available via pip and/or apt-get. Pygeoip is not yet available as a package and must be installed with pip or manually.
+1. Install all of the necessary Python modules listed above. Many of them are available via pip and/or apt-get.
 
-  1. `sudo apt-get install python-crypto python-dpkt python-ipy python-pypcap`
+  * `sudo pip install geoip2 pycrypto dpkt IPy pypcap`
 
-  2. `sudo pip install pygeoip`
-
-2. Configure pygeoip by moving the MaxMind data files (GeoIP.dat, GeoIPv6.dat, GeoIPASNum.dat, GeoIPASNumv6.dat) to <install-location>/share/GeoIP/
+2. Configure GeoIP by moving the MaxMind data files (GeoLite2-Country.mmdb, GeoLite2-ASN.mmdb) to <install-location>/share/GeoIP/
 
 2. Run `make`. This will build Dshell.
 
@@ -50,6 +48,10 @@ Key features:
 ## Development
 * [Using Dshell With PyCharm](doc/UsingDshellWithPyCharm.md)
 
+## Recent Major Updates
+
+* Feb 2019 - Removed deprecated pygeoip dependency, and replaced it with geoip2. This requires the use of new GeoIP data files, listed in the Prerequisites and Installation sections above.
+
 ## Partners
 
 Below are repositories from partners Dshell has worked together with.

docker/Dockerfile

@@ -4,25 +4,25 @@ FROM ubuntu:14.04
 RUN apt-get update && apt-get install -y \
   python-crypto \
   python-dpkt \
-  python-ipy \ 
+  python-ipy \
   python-pypcap \
   python-pip \
+  python-geoip2 \
   wget \
   git
 
-RUN pip install pygeoip
-
 # Download the latest version of the code from GitHub
 WORKDIR /opt/
 RUN git clone https://github.com/USArmyResearchLab/Dshell.git
 
-# download and gunzip GeoIP files
+# download and untar GeoIP files
 WORKDIR /opt/Dshell/share/GeoIP/
-RUN wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
-RUN wget http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz
-RUN wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
-RUN wget http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz
-RUN gunzip *.gz
+RUN wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz
+RUN wget https://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
+RUN tar -zxf GeoLite2-Country.tar.gz
+RUN tar -zxf GeoLite2-ASN.tar.gz
+RUN GeoLite2-Country*/GeoLite2-Country.mmdb .
+RUN GeoLite2-ASN*/GeoLite2-ASN.mmdb .
 
 # make Dshell
 WORKDIR /opt/Dshell/

lib/dshell.py

@@ -14,7 +14,7 @@
 
 # For IP lookups
 try:
-    import pygeoip
+    import geoip2.database
 except:
     pass
 
@@ -106,25 +106,18 @@ def __init__(self, **kwargs):
         self.cleanupts = 0
 
         # instantiate and save references to lookup function
+        geoip_dir = os.path.join(os.environ['DATAPATH'], "GeoIP")
         try:
-            self.geoccdb = [pygeoip.GeoIP(os.environ[
-                                          'DATAPATH'] + '/GeoIP/GeoIP.dat', pygeoip.MEMORY_CACHE).country_code_by_addr]
-            try:
-                self.geoccdb.append(pygeoip.GeoIP(os.environ[
-                                    'DATAPATH'] + '/GeoIP/GeoIPv6.dat', pygeoip.MEMORY_CACHE).country_code_by_addr)
-            except:
-                pass
+            self.geoccdb = geoip2.database.Reader(
+                os.path.join(geoip_dir, "GeoLite2-Country.mmdb")
+            ).country
         except:
             self.geoccdb = None
 
         try:
-            self.geoasndb = [pygeoip.GeoIP(
-                os.environ['DATAPATH'] + '/GeoIP/GeoIPASNum.dat', pygeoip.MEMORY_CACHE).org_by_addr]
-            try:
-                self.geoasndb.append(pygeoip.GeoIP(os.environ[
-                                     'DATAPATH'] + '/GeoIP/GeoIPASNumv6.dat', pygeoip.MEMORY_CACHE).org_by_addr)
-            except:
-                pass
+            self.geoasndb = geoip2.database.Reader(
+                os.path.join(geoip_dir, "GeoLite2-ASN.mmdb")
+            ).asn
         except:
             self.geoasndb = None
 
@@ -250,41 +243,48 @@ class members will be updated from value'''
         if self.name in options:
             self.parseOptions(options[self.name])
 
-    def getGeoIP(self, ip, db=[], notfound='--'):
+    def getGeoIP(self, ip, db=None, notfound='--'):
         """
-        Get record associated with an IP
-        requires GeoIP
+        Get country code associated with an IP.
+        Requires GeoIP library (geoip2) and data files.
         """
-        o = None
-        if db == []:
-            db = self.geoccdb  # default to self.geoccdb
-        for d in db:
-            try:
-                o = d(ip)
-            except:
-                # traceback.print_exc()  # removed by bg on 20121203
-                continue  # passing ipv6 address to ipv4 lookup or v/v
-            if o:
-                return o  # stop when we get a result
-        return notfound
-
-    def getASN(self, ip, db=[], notfound='--'):
+        if not db:
+            db = self.geoccdb
+        try:
+            # Get country code based on order of importance
+            # 1st: Country that owns an IP address registered in another
+            #      location (e.g. military bases in foreign countries)
+            # 2nd: Country in which the IP address is registered
+            # 3rd: Physical country where IP address is located
+            # https://dev.maxmind.com/geoip/geoip2/whats-new-in-geoip2/#Country_Registered_Country_and_Represented_Country
+            location = db(ip)
+            country = (
+                location.represented_country.iso_code or
+                location.registered_country.iso_code or
+                location.country.iso_code or
+                notfound
+            )
+            return country
+        except Exception:
+            # Many expected exceptions can occur here. Ignore them all and
+            # return default value.
+            return notfound
+
+    def getASN(self, ip, db=None, notfound='--'):
         """
-        Get record associated with an IP
-        requires GeoIP
+        Get ASN associated with an IP.
+        Requires GeoIP library (geoip2) and data files.
         """
-        o = None
-        if db == []:
-            db = self.geoasndb  # default to self.geoccdb
-        for d in db:
-            try:
-                o = d(ip)
-            except:
-                # traceback.print_exc()  # removed by bg on 20121203
-                continue  # passing ipv6 address to ipv4 lookup or v/v
-            if o:
-                return o  # stop when we get a result
-        return notfound
+        if not db:
+            db = self.geoasndb
+        try:
+            template = "AS{0.autonomous_system_number} {0.autonomous_system_organization}"
+            asn = template.format( db(ip) )
+            return asn
+        except Exception:
+            # Many expected exceptions can occur here. Ignore them all and
+            # return default value.
+            return notfound
 
     def close(self, conn, ts=None):
         '''for connection based decoders
@@ -851,9 +851,9 @@ def __init__(self, decoder, addr, ts=None, pkt=None, **kwargs):
         # cache
         try:
             self.info(sipcc=decoder.getGeoIP(self.sip, db=decoder.geoccdb),
-                      sipasn=decoder.getGeoIP(self.sip, db=decoder.geoasndb),
+                      sipasn=decoder.getASN(self.sip, db=decoder.geoasndb),
                       dipcc=decoder.getGeoIP(self.dip, db=decoder.geoccdb),
-                      dipasn=decoder.getGeoIP(self.dip, db=decoder.geoasndb))
+                      dipasn=decoder.getASN(self.dip, db=decoder.geoasndb))
         except:
             self.sipcc, self.sipasn, self.dipcc, self.dipasn = None, None, None, None
 

share/GeoIP/readme.txt

@@ -1 +1 @@
-GeoIP Legacy data sets go here.
+GeoIP data sets go here.