diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a657ed --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +rpz.zone +cache/ diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..283de07 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,2 @@ +Andrew Williams +Glen Pitt-Pladdy <> \ No newline at end of file diff --git a/README.md b/README.md index 555a49c..f17ae49 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,11 @@ -# py-hole -A [Pi-hole](https://github.com/pi-hole/pi-hole) inspired DNS firewall / blacklister for use with bind/named using RPZ (plus Laptops running NetworkManger with dnsmasq) +# rpzhole +A [Pi-hole](https://github.com/pi-hole/pi-hole) inspired blacklist RPZ zone generator for Bind 9.8. -For full details see https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/ +This has been forked from [py-hole-bind9RPZ](https://github.com/glenpp/py-hole) and re-wrote to improve runtimes on larger blacklists. -## py-hole-bind9RPZ & py-hole-bind9RPZ_config.yaml -This updates a bind9 RPZ (Response Policy Zone) file against configuration in /etc/bind/py-hole-rpzconfig.yaml -## py-hole-dnsmasq & py-hole-dnsmasq_config.yaml -This is a variant designed for use on Laptops (and other roaming devices) running Mint or Ubuntu that use dnsmasq with NetworkManager. -Since these devices roam, they need local protection as we can't depend on whatever network they are connecting to. - -Default config is coded in, but can be overridden with /etc/py-hole-config.yaml +Further Reading +--------------- +* [Glen Pitt-Pladdy - DNS Firewall (blackhole malicious, like Pi-hole) with bind9](https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/) \ No newline at end of file diff --git a/py-hole-bind9RPZ b/py-hole-bind9RPZ deleted file mode 100755 index 0d5cdf5..0000000 --- a/py-hole-bind9RPZ +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/python -""" -Manages bind9 RPZ file (DNS Firewall) against configured blacklists -Copyright (C) 2017 Glen Pitt-Pladdy - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - - -See: https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/ -""" - - -import yaml -import time -import re -import os -import urllib3 -import sys -import subprocess - - - - - - -# read config -configfile = '/etc/bind/py-hole-bind9RPZ_config.yaml' -config = { - # base config overridden by configfile - 'cachedir': '/var/local/bindRPZ', - 'cacheprefix': 'bindRPZcache-', - 'cacheexpire': 14400, # 4 hours - 'defaultresponse': 'CNAME .', - 'exclusions': {}, - 'blacklists': { - 'StevenBlack': { 'url':'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts', 'format':'hosts', 'hostskey':'0.0.0.0' }, - }, -} -# load yaml file or error -if os.path.isfile ( configfile ): - with open ( configfile, 'r' ) as f: - config.update ( yaml.load(f) ) - # always exclude localhost else we get it blocked for 127.0.0.1 keys - config['exclusions']['localhost'] = True -else: - sys.exit ( "Configuration file %s not found\n" % configfile ) -# at minimum we need to end up with an rpzfile -if 'rpzfile' not in config: - sys.exit ( "Setting for 'rpzfile' not found in configuration %s\n" % configfile ) -# and a template with a serial number -if 'rpztemplate' not in config or not re.search ( r'', config['rpztemplate'] ): - sys.exit ( "Setting for 'rpztemplate' including a serial number marker '' not found in configuration %s\n" % configfile ) -# and a reloadzonecommand: -if 'reloadzonecommand' not in config: - sys.exit ( "Setting for 'reloadzonecommand' not found in configuration %s\n" % configfile ) - - -# build our zone -outputdata = re.sub ( r'', '%010d' % int(time.time()), config['rpztemplate'] ) -seenbefore = {} -commentstart = ';' # for bind -def addcomment ( comment ): - global outputdata - outputdata += "%s%s\n" % (commentstart,comment) -def addhost ( host ): - global outputdata - host = host.lower().strip() - if host in seenbefore: - outputdata += "%s seenbefore in %s %s" % (commentstart,seenbefore[host],commentstart) - if host in config['exclusions']: - outputdata += "%s excluded %s" % (commentstart,commentstart) - outputdata += "%s %s\n" % (host,config['defaultresponse']) - seenbefore[host] = source - - -# grab from web or cache -cacheupto = time.time() - config['cacheexpire'] -if not os.path.isdir ( config['cachedir'] ): - os.makedirs ( config['cachedir'] ) -http = urllib3.PoolManager () -httpheaders = { 'User-Agent': 'py-hole RPZ blackhole manager' } -for source in config['blacklists']: - cachefile = os.path.join ( config['cachedir'], config['cacheprefix'] + source ) - # check cache, download if needed - if os.path.isfile ( cachefile ) and os.path.getmtime ( cachefile ) >= cacheupto: - print "fresh cache %s" % config['blacklists'][source]['url'] - with open ( cachefile, 'rt' ) as f: - data = f.read () - else: - print "retrieve %s" % config['blacklists'][source]['url'] - response = http.request ( 'GET', config['blacklists'][source]['url'], headers=httpheaders ) - if response.status != 200: - sys.exit ( "ERROR - got http response %d for %s" % (response.status,config['blacklists'][source]['url']) ) - # write cache file - with open ( cachefile+'.TMP', 'wt' ) as f: - f.write ( response.data ) - os.rename ( cachefile+'.TMP', cachefile ) - # all done - data = response.data - # we are good to go - outputdata += "\n%s=============================================================================\n" % commentstart - outputdata += "%s Source: %s :: %s\n" % (commentstart,source,config['blacklists'][source]['url']) - outputdata += "%s=============================================================================\n\n" % commentstart - # process data - recordcount = 0 - if config['blacklists'][source]['format'] == 'hosts': - # comments start "#", we only take lines matching "hostskey" - for line in data.splitlines(): - if line == '': - continue - if line[0] == '#': - addcomment ( line[1:] ) - continue - hostlist = re.split ( r'\s+', line ) - if hostlist[0] != config['blacklists'][source]['hostskey']: - # not a matching key - continue - for host in hostlist[1:]: - recordcount += 1 - addhost ( host ) - elif config['blacklists'][source]['format'] == 'raw': - # comments start "#" - for line in data.splitlines(): - if line == '': - continue - if line[0] == '#': - addcomment ( line[1:] ) - continue - host = line.strip() - recordcount += 1 - addhost ( host ) - else: - sys.exit ( "Unknown format %s for %s" % (config['blacklists'][source]['format'],source) ) - if recordcount == 0: - sys.exit ( "Got recordcount of %d for %s" % (recordcount,source) ) - -# if we have a local blacklist, add that also -if 'localblacklist' in config: - outputdata += "\n%s=============================================================================\n" % commentstart - outputdata += "%s Source: Local blacklist from %s\n" % (commentstart,configfile) - outputdata += "%s=============================================================================\n\n" % commentstart - for host in config['localblacklist']: - addhost ( host ) - - -# write the config['rpzfile'] file -with open ( config['rpzfile']+'.TMP', 'wt' ) as f: - f.write ( outputdata ) -os.rename ( config['rpzfile'], config['rpzfile']+'.old' ) -os.rename ( config['rpzfile']+'.TMP', config['rpzfile'] ) -# reload bind zone file -p = subprocess.Popen ( config['reloadzonecommand'], stdin=None, stdout=None ) - diff --git a/py-hole-bind9RPZ_config.yaml b/py-hole-bind9RPZ_config.yaml deleted file mode 100644 index 56ff275..0000000 --- a/py-hole-bind9RPZ_config.yaml +++ /dev/null @@ -1,61 +0,0 @@ ---- - -rpzfile: /etc/bind/db.rpz.example.com -rpztemplate: | - ; see http://www.zytrax.com/books/dns/ch9/rpz.html - ; zone file rpz.example.com - $TTL 2h ; default TTL - $ORIGIN rpz.example.com. - ; email address is never used - @ SOA nonexistent.nodomain.none. dummy.nodomain.none. 12h 15m 3w 2h - ; name server is never accessed but out-of-zone - ; NS nonexistant.nodomain.none - NS boni.example.com. - - ;example.net CNAME . - ;*.example.net CNAME . - - ; Automatic rules start - ; -# end of template - -#cachedir: /var/local/bindRPZ -#cacheprefix: bindRPZcache- -#cacheexpire: 14400 # 4 hours -reloadzonecommand: [ 'rndc', 'reload', 'rpz.example.com' ] -#defaultresponse: CNAME . - - -# see https://github.com/pi-hole/pi-hole/blob/master/adlists.default -# Note: the moment we specify blacklists, the base key completely replaces defaults -blacklists: - StevenBlack: - url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts - format: hosts - hostskey: 0.0.0.0 - malwaredomains: { url: 'https://mirror1.malwaredomains.com/files/justdomains', format: raw } - cameleon: { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' } - abuse.ch: { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' } - disconnect.me_tracking: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' } - disconnect.me_ad: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' } -# hosts-file.net: { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' } -# Windows 10 telemetry: { - securemecca.com: { 'url':'http://securemecca.com/Downloads/hosts.txt', 'format':'hosts', 'hostskey':'127.0.0.1' } -# currently we support formats of: -# * raw -# - considers lines starting "#" as comments -# - one hostname per line -# * hosts -# - considers lines starting "#" as comments -# - requires "hostskey" matching the IP at the start of the line (anything else ignored) -# - multiple hosts per line (typical hosts file with aliases) - -exclusions: - www.googleadservices.com: True # needed for google shopping - pagead.l.doubleclick.net: True # CNAME for www.googleadservices.com needed for google shopping -# Note that "localhost" is always excluded t prevent conflicts - -# we can also add our own local backlist -#localblacklist: -# - evilhost.example.com # going there does evil stuff - diff --git a/py-hole-dnsmasq b/py-hole-dnsmasq deleted file mode 100755 index 73855c8..0000000 --- a/py-hole-dnsmasq +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/python -""" -Manages dnsmasq addn-hosts file (DNS Firewall) against configured blacklists -Copyright (C) 2017 Glen Pitt-Pladdy - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . - - -See: https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/ -""" - - -# removal: delete files specified in cachedir/cacheprefix, dnsmasqblackholeconfig, output - -import yaml -import time -import re -import os -import urllib3 -import sys -import subprocess - - - - - - -# read config -configfile = '/etc/py-hole-dnsmasq_config.yaml' -config = { - # base config overridden by configfile - 'cachedir': '/var/local/py-hole', - 'cacheprefix': 'cache-', - 'cacheexpire': 14400, # 4 hours - 'hostsfile': '/etc/local-hosts-blackhole', - 'dnsmasqblackholeconfig': '/etc/dnsmasq.d/local-hosts-blackhole', - 'defaultresponse': '0.0.0.0', - 'exclusions': { - 'localhost': True, # we need this always else we get it blocked for 127.0.0.1 keys - 'www.googleadservices.com': True, # needed for google shopping - 'pagead.l.doubleclick.net': True, # CNAME for www.googleadservices.com needed for google shopping - }, - 'blacklists': { # see https://github.com/pi-hole/pi-hole/blob/master/adlists.default - 'StevenBlack': { 'url':'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts', 'format':'hosts', 'hostskey':'0.0.0.0' }, - 'malwaredomains': { 'url':'https://mirror1.malwaredomains.com/files/justdomains', 'format':'raw' }, - 'cameleon': { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' }, - 'abuse.ch': { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' }, - 'disconnect.me_tracking': { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' }, - 'disconnect.me_ad': { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' }, -# 'hosts-file.net': { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' }, -# 'Windows 10 telemetry': { - 'securemecca.com': { 'url':'http://securemecca.com/Downloads/hosts.txt', 'format':'hosts', 'hostskey':'127.0.0.1' }, - } -} -# load yaml file or error -if os.path.isfile ( configfile ): - with open ( configfile, 'r' ) as f: - config.update ( yaml.load(f) ) - # always exclude localhost else we get it blocked for 127.0.0.1 keys - config['exclusions']['localhost'] = True -else: - sys.exit ( "Configuration file %s not found\n" % configfile ) - - -# our hostsfile -outputdata = "# created %d\n" % int(time.time()) -seenbefore = {} -commentstart = '#' # for hosts / dnsmasq -def addcomment ( comment ): - global outputdata - outputdata += "%s%s\n" % (commentstart,comment) -def addhost ( host ): - global outputdata - host = host.lower().strip() - if host in seenbefore: - outputdata += "%s seenbefore in %s %s" % (commentstart,seenbefore[host],commentstart) - if host in config['exclusions']: - outputdata += "%s excluded %s" % (commentstart,commentstart) - outputdata += "%s %s\n" % (config['defaultresponse'],host) - seenbefore[host] = source - - -# grab from web or cache -cacheupto = time.time() - config['cacheexpire'] -if not os.path.isdir ( config['cachedir'] ): - os.makedirs ( config['cachedir'] ) -http = urllib3.PoolManager () -httpheaders = { 'User-Agent': 'py-hole hosts blackhole manager' } -for source in config['blacklists']: - cachefile = os.path.join ( config['cachedir'], config['cacheprefix'] + source ) - # check cache, download if needed - if os.path.isfile ( cachefile ) and os.path.getmtime ( cachefile ) >= cacheupto: - print "fresh cache %s" % config['blacklists'][source]['url'] - with open ( cachefile, 'rt' ) as f: - data = f.read () - else: - print "retrieve %s" % config['blacklists'][source]['url'] - response = http.request ( 'GET', config['blacklists'][source]['url'], headers=httpheaders ) - if response.status != 200: - sys.exit ( "ERROR - got http response %d for %s" % (response.status,config['blacklists'][source]['url']) ) - # write cache file - with open ( cachefile+'.TMP', 'wt' ) as f: - f.write ( response.data ) - os.rename ( cachefile+'.TMP', cachefile ) - # all done - data = response.data - # we are good to go - outputdata += "\n%s=============================================================================\n" % commentstart - outputdata += "%s Source: %s :: %s\n" % (commentstart,source,config['blacklists'][source]['url']) - outputdata += "%s=============================================================================\n\n" % commentstart - # process data - recordcount = 0 - if config['blacklists'][source]['format'] == 'hosts': - # comments start "#", we only take lines matching "hostskey" - for line in data.splitlines(): - if line == '': - continue - if line[0] == '#': - addcomment ( line[1:] ) - continue - hostlist = re.split ( r'\s+', line ) - if hostlist[0] != config['blacklists'][source]['hostskey']: - # not a matching key - continue - for host in hostlist[1:]: - recordcount += 1 - addhost ( host ) - elif config['blacklists'][source]['format'] == 'raw': - # comments start "#" - for line in data.splitlines(): - if line == '': - continue - if line[0] == '#': - addcomment ( line[1:] ) - continue - host = line.strip() - recordcount += 1 - addhost ( host ) - else: - sys.exit ( "Unknown format %s for %s" % (config['blacklists'][source]['format'],source) ) - if recordcount == 0: - sys.exit ( "Got recordcount of %d for %s" % (recordcount,source) ) - -# if we have a local blacklist, add that also -if 'localblacklist' in config: - outputdata += "\n%s=============================================================================\n" % commentstart - outputdata += "%s Source: Local blacklist from %s\n" % (commentstart,configfile) - outputdata += "%s=============================================================================\n\n" % commentstart - for host in config['localblacklist']: - addhost ( host ) - - -# write the config['hostsfile'] file -with open ( config['hostsfile']+'.TMP', 'wt' ) as f: - f.write ( outputdata ) -os.rename ( config['hostsfile'], config['hostsfile']+'.old' ) -os.rename ( config['hostsfile']+'.TMP', config['hostsfile'] ) - - -# ensure we have a dnsmasq config file - we assume if it's there it's sufficient TODO maybe we should check -if not os.path.isfile ( config['dnsmasqblackholeconfig'] ): - with open ( config['dnsmasqblackholeconfig']+'.TMP', 'wt' ) as f: - f.write ( "addn-hosts=%s\n" % output ) - os.rename ( config['dnsmasqblackholeconfig']+'.TMP', config['dnsmasqblackholeconfig'] ) -# TODO reload dnsmasq (SIGHUP re-reads files, but not config) - diff --git a/py-hole-dnsmasq_config.yaml b/py-hole-dnsmasq_config.yaml deleted file mode 100644 index 4aadb77..0000000 --- a/py-hole-dnsmasq_config.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -#cachedir: /var/local/py-hole -#cacheprefix: cache- -#cacheexpire: 14400 # 4 hours -#defaultresponse: 0.0.0.0 - -hostsfile: /etc/local-hosts-blackhole -dnsmasqblackholeconfig: /etc/dnsmasq.d/local-hosts-blackhole - - -# see https://github.com/pi-hole/pi-hole/blob/master/adlists.default -# Note: the moment we specify blacklists, the base key completely replaces defaults -blacklists: - StevenBlack: - url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts - format: hosts - hostskey: 0.0.0.0 - malwaredomains: { url: 'https://mirror1.malwaredomains.com/files/justdomains', format: raw } - cameleon: { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' } - abuse.ch: { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' } - disconnect.me_tracking: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' } - disconnect.me_ad: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' } -# hosts-file.net: { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' }, -# Windows 10 telemetry: { - securemecca.com: { url: 'http://securemecca.com/Downloads/hosts.txt', format: hosts, hostskey: 127.0.0.1 } -# currently we support formats of: -# * raw -# - considers lines starting "#" as comments -# - one hostname per line -# * hosts -# - considers lines starting "#" as comments -# - requires "hostskey" matching the IP at the start of the line (anything else ignored) -# - multiple hosts per line (typical hosts file with aliases) - -exclusions: - www.googleadservices.com: True # needed for google shopping - pagead.l.doubleclick.net: True # CNAME for www.googleadservices.com needed for google shopping -# Note that "localhost" is always excluded t prevent conflicts - -# we can also add our own local backlist -#localblacklist: -# - evilhost.example.com # going there does evil stuff - diff --git a/rpzhole b/rpzhole new file mode 100755 index 0000000..27b7ce2 --- /dev/null +++ b/rpzhole @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +import logging +import yaml +import time +import re +import os +import sys +import requests +import argparse +import subprocess + +__version__ = '0.1' + +_logger = logging.getLogger("rpzhole") + +def download_blacklist(url, cache_file): + """Downloads and stores a blacklist, returns a filepath""" + + # If we have an existing file, do a header call, check the size and if its the same don't re-download + if os.path.exists(cache_file): + resp = requests.head(url) + if resp.ok and 'content-length' in resp.headers and os.path.getsize(cache_file) == int(resp.headers['content-length']): + return True + + # If we don't have the file or missed the cache, download the file + resp = requests.get(url, headers={'User-Agent': 'rpzhole %s' % __version__}) + if resp.ok: + with file(cache_file, 'wb') as fobj: + for block in resp.iter_content(1024): + if block: fobj.write(block) + return True + +def parse_blacklist(format, filename=None, fobj=None): + if not format in ('hosts', 'raw'): + raise Exception('Unsupported format %s' % format) + if not fobj: + with open(filename, 'rb') as fobj: + return parse_blacklist(format, fobj=fobj) + else: + data = [] + if format == 'hosts': + for line in fobj: + if line == '' or line[0] == '#': continue + items = re.split ( r'\s+', line ) + data.extend(items[1:]) + elif format == 'raw': + for line in fobj: + if line == '' or line[0] == '#': continue + data.append(line.strip()) + return data + +def write_rpz(hosts, filename, origin='rpz.black.hole'): + rpz_header = ["$TTL 60", "$ORIGIN %s" % origin, "@ SOA nonexistent.nodomain.none. dummy.nodomain.none. %d 12h 15m 3w 2h" % time.time(), "NS nonexistant.nodomain.none"] + with open(filename, 'wb') as fobj: + for line in rpz_header: + fobj.write('%s\n' % line) + fobj.write('\n') + for host in hosts: + if host: + fobj.write("%s\t\tA\t127.0.0.1\n" % host) + +def main(): + + # Parse command line arguments + parser = argparse.ArgumentParser(description='Generate Bind9 RPZ zone files from Blacklists') + parser.add_argument('--config', dest='config_file') + parser.add_argument('--debug', help='Enables logging of debug messages', default=False, action='store_true') + parser.add_argument('--silent', help='Disables all logging except for errors, useful for cron execution', default=False, action='store_true') + parser.add_argument('--reloadzone', help='Ask Bind to reload the RPZ zone file once completed', default=False, action='store_true') + args = parser.parse_args() + + # Enable verbosity + if args.silent: + level = logging.ERROR + elif args.debug: + level = logging.DEBUG + else: + level = logging.INFO + logging.basicConfig(level=level) + logging.getLogger("requests").setLevel(logging.WARNING) + _logger.debug('Parsed Args: %s', str(args)) + + # Load configuration + if not args.config_file: + default_paths = [os.path.join(os.getcwd(), 'rpzhole.yaml'), os.path.expanduser('~/.rpzhole.yaml'), '/etc/bind/rpzhole.yaml', '/etc/rpzhole.yaml'] + for filepath in default_paths: + if os.path.exists(filepath): + _logger.debug('Found configuration file %s', filepath) + args.config_file = filepath + break + else: + _logger.error("Unable to find usable configuration file in any of the following locations: %s", ', '.join(default_paths)) + sys.exit(1) + config_file_path = os.path.abspath(os.path.expanduser(args.config_file)) + _logger.debug('Loading configuration file %s', config_file_path) + with open(config_file_path, 'rb') as fobj: + config = yaml.load(fobj) + + # Pre-create the cache locations + if not os.path.exists(config['cache_dir']): + try: + os.makedirs(config['cache_dir']) + except OSError as e: + _logger.error('Unable to create caching directory: %s, exiting...', e) + sys.exit(1) + + # Download blacklists + blacklist_hosts = [] + for name, val in config['blacklists'].items(): + _logger.info('Processing %s blacklist', name) + cache_file = os.path.join(config['cache_dir'], config['cache_prefix'] + name + '.list') + try: + if download_blacklist(val['url'], cache_file): + _logger.debug('Parsing %s as format %s', name, val['format']) + list_hosts = parse_blacklist(val['format'], cache_file) + _logger.info('Adding %d hosts from %s blacklist', len(list_hosts), name) + blacklist_hosts.extend(list_hosts) + except Exception as e: + _logger.error('Unable to download or parse %s blacklist: %s', name, e) + + # Remove duplicates and exclude any hosts on the exclusion list + output_hostlist = set(blacklist_hosts) - set(config['exclusions']) + _logger.info('%d unique hosts used to create RPZ, %d entries from blacklists, %d exclusion hosts', len(output_hostlist), len(blacklist_hosts), len(config['exclusions'])) + + # write RPZ + write_rpz(output_hostlist, config['output_filename'], config['origin']) + _logger.info('RPZ zone wrote to %s', config['output_filename']) + + # reload RPZ + if args.reloadzone: + _logger.info('Informing Bind to reload the zone') + res = subprocess.call('rndc reload %s' % config['origin']) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/rpzhole.yaml b/rpzhole.yaml new file mode 100644 index 0000000..e0d88d8 --- /dev/null +++ b/rpzhole.yaml @@ -0,0 +1,34 @@ +--- + +blacklists: + StevenBlack: + url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts + format: hosts + malwaredomains: + url: 'https://mirror1.malwaredomains.com/files/justdomains' + format: raw + cameleon: + url: http://sysctl.org/cameleon/hosts + format: hosts + abuse.ch: + url: https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist + format: raw + disconnect.me_tracking: + url: https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt + format: raw + disconnect.me_ad: + url: https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt + format: raw + securemecca.com: + url: http://securemecca.com/Downloads/hosts.txt + format: hosts + +exclusions: + - www.googleadservices.com + - pagead.l.doubleclick.net + +origin: rpz.home.tensixtyone.com +output_filename: ./rpz.zone +cache_dir: ./cache/ +cache_prefix: rpzhole- +