Ticket #12872: 0001-Added-countryCode-attribute-refactored-GeoIP-logic-i.patch

File 0001-Added-countryCode-attribute-refactored-GeoIP-logic-i.patch, 7.7 KB (added by pagea, 5 years ago)

Bridges now have a countryCode attribute. In addition, I moved all GeoIP logic into its own module in bridgedb.geo.

  • lib/bridgedb/Bridges.py

    From dd08b8800afd10748cde722fc0588162f0e450e2 Mon Sep 17 00:00:00 2001
    From: "Alden S. Page" <pagea@allegheny.edu>
    Date: Thu, 1 Jan 2015 21:24:09 -0500
    Subject: [PATCH] Added countryCode attribute, refactored GeoIP logic into its
     own module.
    
    Instances of Bridge now keep track of their country codes. This should help us
    hand out bridges more intelligently. In particular, we will soon be able to
    withold bridges from .ir and .sy as per issue #12843. Since multiple modules
    make use of geoip (HTTPServer and Bridges), I moved the geoip logic out of
    HTTPServer and into its own module called geo.py.
    ---
     lib/bridgedb/Bridges.py    | 11 +++++++
     lib/bridgedb/HTTPServer.py | 29 +++----------------
     lib/bridgedb/geo.py        | 71 ++++++++++++++++++++++++++++++++++++++++++++++
     3 files changed, 86 insertions(+), 25 deletions(-)
     create mode 100644 lib/bridgedb/geo.py
    
    diff --git a/lib/bridgedb/Bridges.py b/lib/bridgedb/Bridges.py
    index 2439130..4b80f12 100644
    a b import random 
    2222
    2323import bridgedb.Storage
    2424import bridgedb.Bucket
     25import bridgedb.geo
    2526
    2627from bridgedb.crypto import getHMACFunc
    2728from bridgedb.parse import addr
    class Bridge(object): 
    118119        given in the bridge's descriptor, corresponding to desc_digest.
    119120    :attr bool verified: Did we receive the descriptor for this bridge that
    120121        was specified in the networkstatus?
     122    :attr str countryCode: The two-letter country code of this bridge as
     123    reported by GeoIP.
    121124    """
    122125    def __init__(self, nickname, ip, orport, fingerprint=None, id_digest=None,
    123126                 or_addresses=None, transports=None):
    class Bridge(object): 
    135138        self.desc_digest = None
    136139        self.ei_digest = None
    137140        self.verified = False
     141        self.countryCode = None
    138142
    139143        if id_digest is not None:
    140144            assert fingerprint is None
    class Bridge(object): 
    149153        else:
    150154            raise TypeError("Bridge with no ID")
    151155
     156        # Determine the country code of this bridge.
     157        self.countryCode = bridgedb.geo.getCountryCode(ip)
     158        if self.countryCode is None:
     159            logging.debug("  Bridge: Country code not detected. IP: %s" % ip)
     160        else:
     161            logging.debug("  Bridge: Country code found: %s" % self.countryCode)
     162
    152163    def setDescriptorDigest(self, digest):
    153164        """Set the descriptor digest, specified in the NS."""
    154165        self.desc_digest = digest
  • lib/bridgedb/HTTPServer.py

    diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py
    index 464ae0d..54d9112 100644
    a b from functools import partial 
    2424
    2525from ipaddr import IPv4Address
    2626from ipaddr import IPv6Address
     27from ipaddr import IPAddress
    2728
    2829import mako.exceptions
    2930from mako.template import Template
    from twisted.web import static 
    3839from twisted.web.util import redirectTo
    3940
    4041import bridgedb.Dist
     42import bridgedb.geo
    4143
    4244from bridgedb import captcha
    4345from bridgedb import crypto
    from bridgedb.safelog import logSafely 
    5355
    5456
    5557TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), 'templates')
    56 GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat'
    5758rtl_langs = ('ar', 'he', 'fa', 'gu_IN', 'ku')
    5859
    5960# Setting `filesystem_checks` to False is recommended for production servers,
    lookup = TemplateLookup(directories=[TEMPLATE_DIR], 
    6970                        collection_size=500)
    7071logging.debug("Set template root to %s" % TEMPLATE_DIR)
    7172
    72 try:
    73     # Make sure we have the database before trying to import the module:
    74     if not os.path.isfile(GEOIP_DBFILE):  # pragma: no cover
    75         raise EnvironmentError("Could not find %r. On Debian-based systems, "\
    76                                "please install the geoip-database package."
    77                                % GEOIP_DBFILE)
    78     # This is a "pure" python version which interacts with the Maxmind GeoIP
    79     # API (version 1). It requires, in Debian, the libgeoip-dev and
    80     # geoip-database packages.
    81     import pygeoip
    82     geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE)
    83     logging.info("GeoIP database loaded")
    84 except Exception as err:  # pragma: no cover
    85     logging.warn("Error while loading geoip module: %r" % err)
    86     geoip = None
    87 
    88 
    8973def replaceErrorPage(error, template_name=None):
    9074    """Create a general error page for displaying in place of tracebacks.
    9175
    class WebResourceBridges(resource.Resource): 
    689673        else:
    690674            ip = request.getClientIP()
    691675
    692         # XXX This can also be a separate function
    693         # XXX if the ip is None, this throws an exception
    694         if geoip:
    695             countryCode = geoip.country_code_by_addr(ip)
    696             if countryCode:
    697                 logging.debug("Client request from GeoIP CC: %s" % countryCode)
     676        # Record what country the client is in.
     677        countryCode = bridgedb.geo.getCountryCode(IPAddress(ip))
    698678
    699679        # XXX separate function again
    700680        format = request.args.get("format", None)
    class WebResourceBridges(resource.Resource): 
    798778
    799779        return rendered
    800780
    801 
    802781class WebRoot(resource.Resource):
    803782    """The parent resource of all other documents hosted by the webserver."""
    804783
  • new file lib/bridgedb/geo.py

    diff --git a/lib/bridgedb/geo.py b/lib/bridgedb/geo.py
    new file mode 100644
    index 0000000..b2be72c
    - +  
     1#
     2#
     3# This file is part of BridgeDB, a Tor bridge distribution system.
     4#
     5# :authors: see AUTHORS file
     6# :copyright: (c) 2007-2015, The Tor Project, Inc.
     7# :license: 3-Clause BSD, see LICENSE for licensing information
     8
     9"""
     10Boilerplate setup for GeoIP. GeoIP allows us to look up the country code
     11associated with an IP address. This is a "pure" python version which interacts
     12with the Maxmind GeoIP API (version 1). It requires, in Debian, the libgeoip-dev
     13and geoip-database packages.
     14"""
     15
     16import logging
     17from os.path import isfile
     18
     19from bridgedb.safelog import logSafely
     20from ipaddr import IPv4Address, IPv6Address
     21
     22# IPv4 database
     23GEOIP_DBFILE = '/usr/share/GeoIP/GeoIP.dat'
     24# IPv6 database
     25GEOIPv6_DBFILE = '/usr/share/GeoIP/GeoIPv6.dat'
     26try:
     27    # Make sure we have the database before trying to import the module:
     28    if not (isfile(GEOIP_DBFILE) and isfile(GEOIPv6_DBFILE)):
     29        raise EnvironmentError("Could not find %r. On Debian-based systems, "\
     30                               "please install the geoip-database package."
     31                               % GEOIP_DBFILE)
     32
     33    import pygeoip
     34    geoip = pygeoip.GeoIP(GEOIP_DBFILE, flags=pygeoip.MEMORY_CACHE)
     35    geoipv6 = pygeoip.GeoIP(GEOIPv6_DBFILE, flags=pygeoip.MEMORY_CACHE)
     36    logging.info("GeoIP databases loaded")
     37except Exception as err:  # pragma: no cover
     38    logging.warn("Error while loading geoip module: %r" % err)
     39    geoip = None
     40
     41def getCountryCode(IPAddr):
     42    """Returns the two-letter country code of a given IP address.
     43
     44    :param IPAddr: (:class:`ipaddr.IPAddress`) An IPv4 OR IPv6 address.
     45    """
     46    ip = None
     47    version = None
     48    try:
     49        ip = IPAddr.exploded
     50        version = IPAddr.version
     51    except AttributeError as err:
     52        logging.warn("Wrong type passed to getCountryCode. Offending call:"
     53        " %r" % err)
     54        return None
     55
     56    # GeoIP has two databases: one for IPv4 addresses, and one for IPv6
     57    # addresses. This will ensure we use the correct one.
     58    db = None   
     59    if version == 4:
     60        db = geoip
     61    else:
     62        db = geoipv6
     63
     64    # Look up the country code of the address.
     65    countryCode = db.country_code_by_addr(ip)
     66    if countryCode:
     67        logging.debug("Looked up country code: %s" % countryCode)
     68        return countryCode
     69    else:
     70        logging.debug("Country code was not detected. IP: %s" % ip)
     71        return None