mirror of
https://github.com/nikdoof/rpzhole.git
synced 2025-12-13 06:42:18 +00:00
Rewrite/rename, initial v0.1 non packaged version
This commit is contained in:
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
rpz.zone
|
||||
cache/
|
||||
2
AUTHORS
Normal file
2
AUTHORS
Normal file
@@ -0,0 +1,2 @@
|
||||
Andrew Williams <andy@tensixtyone.com>
|
||||
Glen Pitt-Pladdy <>
|
||||
16
README.md
16
README.md
@@ -1,15 +1,11 @@
|
||||
# py-hole
|
||||
A [Pi-hole](https://github.com/pi-hole/pi-hole) inspired DNS firewall / blacklister for use with bind/named using RPZ (plus Laptops running NetworkManger with dnsmasq)
|
||||
# rpzhole
|
||||
A [Pi-hole](https://github.com/pi-hole/pi-hole) inspired blacklist RPZ zone generator for Bind 9.8.
|
||||
|
||||
For full details see https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/
|
||||
This has been forked from [py-hole-bind9RPZ](https://github.com/glenpp/py-hole) and re-wrote to improve runtimes on larger blacklists.
|
||||
|
||||
## py-hole-bind9RPZ & py-hole-bind9RPZ_config.yaml
|
||||
This updates a bind9 RPZ (Response Policy Zone) file against configuration in /etc/bind/py-hole-rpzconfig.yaml
|
||||
|
||||
## py-hole-dnsmasq & py-hole-dnsmasq_config.yaml
|
||||
This is a variant designed for use on Laptops (and other roaming devices) running Mint or Ubuntu that use dnsmasq with NetworkManager.
|
||||
|
||||
Since these devices roam, they need local protection as we can't depend on whatever network they are connecting to.
|
||||
|
||||
Default config is coded in, but can be overridden with /etc/py-hole-config.yaml
|
||||
Further Reading
|
||||
---------------
|
||||
|
||||
* [Glen Pitt-Pladdy - DNS Firewall (blackhole malicious, like Pi-hole) with bind9](https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/)
|
||||
164
py-hole-bind9RPZ
164
py-hole-bind9RPZ
@@ -1,164 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
"""
|
||||
Manages bind9 RPZ file (DNS Firewall) against configured blacklists
|
||||
Copyright (C) 2017 Glen Pitt-Pladdy
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
See: https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/
|
||||
"""
|
||||
|
||||
|
||||
import yaml
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import urllib3
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# read config
|
||||
configfile = '/etc/bind/py-hole-bind9RPZ_config.yaml'
|
||||
config = {
|
||||
# base config overridden by configfile
|
||||
'cachedir': '/var/local/bindRPZ',
|
||||
'cacheprefix': 'bindRPZcache-',
|
||||
'cacheexpire': 14400, # 4 hours
|
||||
'defaultresponse': 'CNAME .',
|
||||
'exclusions': {},
|
||||
'blacklists': {
|
||||
'StevenBlack': { 'url':'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts', 'format':'hosts', 'hostskey':'0.0.0.0' },
|
||||
},
|
||||
}
|
||||
# load yaml file or error
|
||||
if os.path.isfile ( configfile ):
|
||||
with open ( configfile, 'r' ) as f:
|
||||
config.update ( yaml.load(f) )
|
||||
# always exclude localhost else we get it blocked for 127.0.0.1 keys
|
||||
config['exclusions']['localhost'] = True
|
||||
else:
|
||||
sys.exit ( "Configuration file %s not found\n" % configfile )
|
||||
# at minimum we need to end up with an rpzfile
|
||||
if 'rpzfile' not in config:
|
||||
sys.exit ( "Setting for 'rpzfile' not found in configuration %s\n" % configfile )
|
||||
# and a template with a serial number
|
||||
if 'rpztemplate' not in config or not re.search ( r'<SERIAL>', config['rpztemplate'] ):
|
||||
sys.exit ( "Setting for 'rpztemplate' including a serial number marker '<SERIAL>' not found in configuration %s\n" % configfile )
|
||||
# and a reloadzonecommand:
|
||||
if 'reloadzonecommand' not in config:
|
||||
sys.exit ( "Setting for 'reloadzonecommand' not found in configuration %s\n" % configfile )
|
||||
|
||||
|
||||
# build our zone
|
||||
outputdata = re.sub ( r'<SERIAL>', '%010d' % int(time.time()), config['rpztemplate'] )
|
||||
seenbefore = {}
|
||||
commentstart = ';' # for bind
|
||||
def addcomment ( comment ):
|
||||
global outputdata
|
||||
outputdata += "%s%s\n" % (commentstart,comment)
|
||||
def addhost ( host ):
|
||||
global outputdata
|
||||
host = host.lower().strip()
|
||||
if host in seenbefore:
|
||||
outputdata += "%s seenbefore in %s %s" % (commentstart,seenbefore[host],commentstart)
|
||||
if host in config['exclusions']:
|
||||
outputdata += "%s excluded %s" % (commentstart,commentstart)
|
||||
outputdata += "%s %s\n" % (host,config['defaultresponse'])
|
||||
seenbefore[host] = source
|
||||
|
||||
|
||||
# grab from web or cache
|
||||
cacheupto = time.time() - config['cacheexpire']
|
||||
if not os.path.isdir ( config['cachedir'] ):
|
||||
os.makedirs ( config['cachedir'] )
|
||||
http = urllib3.PoolManager ()
|
||||
httpheaders = { 'User-Agent': 'py-hole RPZ blackhole manager' }
|
||||
for source in config['blacklists']:
|
||||
cachefile = os.path.join ( config['cachedir'], config['cacheprefix'] + source )
|
||||
# check cache, download if needed
|
||||
if os.path.isfile ( cachefile ) and os.path.getmtime ( cachefile ) >= cacheupto:
|
||||
print "fresh cache %s" % config['blacklists'][source]['url']
|
||||
with open ( cachefile, 'rt' ) as f:
|
||||
data = f.read ()
|
||||
else:
|
||||
print "retrieve %s" % config['blacklists'][source]['url']
|
||||
response = http.request ( 'GET', config['blacklists'][source]['url'], headers=httpheaders )
|
||||
if response.status != 200:
|
||||
sys.exit ( "ERROR - got http response %d for %s" % (response.status,config['blacklists'][source]['url']) )
|
||||
# write cache file
|
||||
with open ( cachefile+'.TMP', 'wt' ) as f:
|
||||
f.write ( response.data )
|
||||
os.rename ( cachefile+'.TMP', cachefile )
|
||||
# all done
|
||||
data = response.data
|
||||
# we are good to go
|
||||
outputdata += "\n%s=============================================================================\n" % commentstart
|
||||
outputdata += "%s Source: %s :: %s\n" % (commentstart,source,config['blacklists'][source]['url'])
|
||||
outputdata += "%s=============================================================================\n\n" % commentstart
|
||||
# process data
|
||||
recordcount = 0
|
||||
if config['blacklists'][source]['format'] == 'hosts':
|
||||
# comments start "#", we only take lines matching "hostskey"
|
||||
for line in data.splitlines():
|
||||
if line == '':
|
||||
continue
|
||||
if line[0] == '#':
|
||||
addcomment ( line[1:] )
|
||||
continue
|
||||
hostlist = re.split ( r'\s+', line )
|
||||
if hostlist[0] != config['blacklists'][source]['hostskey']:
|
||||
# not a matching key
|
||||
continue
|
||||
for host in hostlist[1:]:
|
||||
recordcount += 1
|
||||
addhost ( host )
|
||||
elif config['blacklists'][source]['format'] == 'raw':
|
||||
# comments start "#"
|
||||
for line in data.splitlines():
|
||||
if line == '':
|
||||
continue
|
||||
if line[0] == '#':
|
||||
addcomment ( line[1:] )
|
||||
continue
|
||||
host = line.strip()
|
||||
recordcount += 1
|
||||
addhost ( host )
|
||||
else:
|
||||
sys.exit ( "Unknown format %s for %s" % (config['blacklists'][source]['format'],source) )
|
||||
if recordcount == 0:
|
||||
sys.exit ( "Got recordcount of %d for %s" % (recordcount,source) )
|
||||
|
||||
# if we have a local blacklist, add that also
|
||||
if 'localblacklist' in config:
|
||||
outputdata += "\n%s=============================================================================\n" % commentstart
|
||||
outputdata += "%s Source: Local blacklist from %s\n" % (commentstart,configfile)
|
||||
outputdata += "%s=============================================================================\n\n" % commentstart
|
||||
for host in config['localblacklist']:
|
||||
addhost ( host )
|
||||
|
||||
|
||||
# write the config['rpzfile'] file
|
||||
with open ( config['rpzfile']+'.TMP', 'wt' ) as f:
|
||||
f.write ( outputdata )
|
||||
os.rename ( config['rpzfile'], config['rpzfile']+'.old' )
|
||||
os.rename ( config['rpzfile']+'.TMP', config['rpzfile'] )
|
||||
# reload bind zone file
|
||||
p = subprocess.Popen ( config['reloadzonecommand'], stdin=None, stdout=None )
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
---
|
||||
|
||||
rpzfile: /etc/bind/db.rpz.example.com
|
||||
rpztemplate: |
|
||||
; see http://www.zytrax.com/books/dns/ch9/rpz.html
|
||||
; zone file rpz.example.com
|
||||
$TTL 2h ; default TTL
|
||||
$ORIGIN rpz.example.com.
|
||||
; email address is never used
|
||||
@ SOA nonexistent.nodomain.none. dummy.nodomain.none. <SERIAL> 12h 15m 3w 2h
|
||||
; name server is never accessed but out-of-zone
|
||||
; NS nonexistant.nodomain.none
|
||||
NS boni.example.com.
|
||||
|
||||
;example.net CNAME .
|
||||
;*.example.net CNAME .
|
||||
|
||||
; Automatic rules start
|
||||
;
|
||||
# end of template
|
||||
|
||||
#cachedir: /var/local/bindRPZ
|
||||
#cacheprefix: bindRPZcache-
|
||||
#cacheexpire: 14400 # 4 hours
|
||||
reloadzonecommand: [ 'rndc', 'reload', 'rpz.example.com' ]
|
||||
#defaultresponse: CNAME .
|
||||
|
||||
|
||||
# see https://github.com/pi-hole/pi-hole/blob/master/adlists.default
|
||||
# Note: the moment we specify blacklists, the base key completely replaces defaults
|
||||
blacklists:
|
||||
StevenBlack:
|
||||
url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
|
||||
format: hosts
|
||||
hostskey: 0.0.0.0
|
||||
malwaredomains: { url: 'https://mirror1.malwaredomains.com/files/justdomains', format: raw }
|
||||
cameleon: { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' }
|
||||
abuse.ch: { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' }
|
||||
disconnect.me_tracking: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' }
|
||||
disconnect.me_ad: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' }
|
||||
# hosts-file.net: { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' }
|
||||
# Windows 10 telemetry: {
|
||||
securemecca.com: { 'url':'http://securemecca.com/Downloads/hosts.txt', 'format':'hosts', 'hostskey':'127.0.0.1' }
|
||||
# currently we support formats of:
|
||||
# * raw
|
||||
# - considers lines starting "#" as comments
|
||||
# - one hostname per line
|
||||
# * hosts
|
||||
# - considers lines starting "#" as comments
|
||||
# - requires "hostskey" matching the IP at the start of the line (anything else ignored)
|
||||
# - multiple hosts per line (typical hosts file with aliases)
|
||||
|
||||
exclusions:
|
||||
www.googleadservices.com: True # needed for google shopping
|
||||
pagead.l.doubleclick.net: True # CNAME for www.googleadservices.com needed for google shopping
|
||||
# Note that "localhost" is always excluded t prevent conflicts
|
||||
|
||||
# we can also add our own local backlist
|
||||
#localblacklist:
|
||||
# - evilhost.example.com # going there does evil stuff
|
||||
|
||||
177
py-hole-dnsmasq
177
py-hole-dnsmasq
@@ -1,177 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
"""
|
||||
Manages dnsmasq addn-hosts file (DNS Firewall) against configured blacklists
|
||||
Copyright (C) 2017 Glen Pitt-Pladdy
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
See: https://www.pitt-pladdy.com/blog/_20170407-105402_0100_DNS_Firewall_blackhole_malicious_like_Pi-hole_with_bind9/
|
||||
"""
|
||||
|
||||
|
||||
# removal: delete files specified in cachedir/cacheprefix, dnsmasqblackholeconfig, output
|
||||
|
||||
import yaml
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import urllib3
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# read config
|
||||
configfile = '/etc/py-hole-dnsmasq_config.yaml'
|
||||
config = {
|
||||
# base config overridden by configfile
|
||||
'cachedir': '/var/local/py-hole',
|
||||
'cacheprefix': 'cache-',
|
||||
'cacheexpire': 14400, # 4 hours
|
||||
'hostsfile': '/etc/local-hosts-blackhole',
|
||||
'dnsmasqblackholeconfig': '/etc/dnsmasq.d/local-hosts-blackhole',
|
||||
'defaultresponse': '0.0.0.0',
|
||||
'exclusions': {
|
||||
'localhost': True, # we need this always else we get it blocked for 127.0.0.1 keys
|
||||
'www.googleadservices.com': True, # needed for google shopping
|
||||
'pagead.l.doubleclick.net': True, # CNAME for www.googleadservices.com needed for google shopping
|
||||
},
|
||||
'blacklists': { # see https://github.com/pi-hole/pi-hole/blob/master/adlists.default
|
||||
'StevenBlack': { 'url':'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts', 'format':'hosts', 'hostskey':'0.0.0.0' },
|
||||
'malwaredomains': { 'url':'https://mirror1.malwaredomains.com/files/justdomains', 'format':'raw' },
|
||||
'cameleon': { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' },
|
||||
'abuse.ch': { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' },
|
||||
'disconnect.me_tracking': { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' },
|
||||
'disconnect.me_ad': { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' },
|
||||
# 'hosts-file.net': { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' },
|
||||
# 'Windows 10 telemetry': {
|
||||
'securemecca.com': { 'url':'http://securemecca.com/Downloads/hosts.txt', 'format':'hosts', 'hostskey':'127.0.0.1' },
|
||||
}
|
||||
}
|
||||
# load yaml file or error
|
||||
if os.path.isfile ( configfile ):
|
||||
with open ( configfile, 'r' ) as f:
|
||||
config.update ( yaml.load(f) )
|
||||
# always exclude localhost else we get it blocked for 127.0.0.1 keys
|
||||
config['exclusions']['localhost'] = True
|
||||
else:
|
||||
sys.exit ( "Configuration file %s not found\n" % configfile )
|
||||
|
||||
|
||||
# our hostsfile
|
||||
outputdata = "# created %d\n" % int(time.time())
|
||||
seenbefore = {}
|
||||
commentstart = '#' # for hosts / dnsmasq
|
||||
def addcomment ( comment ):
|
||||
global outputdata
|
||||
outputdata += "%s%s\n" % (commentstart,comment)
|
||||
def addhost ( host ):
|
||||
global outputdata
|
||||
host = host.lower().strip()
|
||||
if host in seenbefore:
|
||||
outputdata += "%s seenbefore in %s %s" % (commentstart,seenbefore[host],commentstart)
|
||||
if host in config['exclusions']:
|
||||
outputdata += "%s excluded %s" % (commentstart,commentstart)
|
||||
outputdata += "%s %s\n" % (config['defaultresponse'],host)
|
||||
seenbefore[host] = source
|
||||
|
||||
|
||||
# grab from web or cache
|
||||
cacheupto = time.time() - config['cacheexpire']
|
||||
if not os.path.isdir ( config['cachedir'] ):
|
||||
os.makedirs ( config['cachedir'] )
|
||||
http = urllib3.PoolManager ()
|
||||
httpheaders = { 'User-Agent': 'py-hole hosts blackhole manager' }
|
||||
for source in config['blacklists']:
|
||||
cachefile = os.path.join ( config['cachedir'], config['cacheprefix'] + source )
|
||||
# check cache, download if needed
|
||||
if os.path.isfile ( cachefile ) and os.path.getmtime ( cachefile ) >= cacheupto:
|
||||
print "fresh cache %s" % config['blacklists'][source]['url']
|
||||
with open ( cachefile, 'rt' ) as f:
|
||||
data = f.read ()
|
||||
else:
|
||||
print "retrieve %s" % config['blacklists'][source]['url']
|
||||
response = http.request ( 'GET', config['blacklists'][source]['url'], headers=httpheaders )
|
||||
if response.status != 200:
|
||||
sys.exit ( "ERROR - got http response %d for %s" % (response.status,config['blacklists'][source]['url']) )
|
||||
# write cache file
|
||||
with open ( cachefile+'.TMP', 'wt' ) as f:
|
||||
f.write ( response.data )
|
||||
os.rename ( cachefile+'.TMP', cachefile )
|
||||
# all done
|
||||
data = response.data
|
||||
# we are good to go
|
||||
outputdata += "\n%s=============================================================================\n" % commentstart
|
||||
outputdata += "%s Source: %s :: %s\n" % (commentstart,source,config['blacklists'][source]['url'])
|
||||
outputdata += "%s=============================================================================\n\n" % commentstart
|
||||
# process data
|
||||
recordcount = 0
|
||||
if config['blacklists'][source]['format'] == 'hosts':
|
||||
# comments start "#", we only take lines matching "hostskey"
|
||||
for line in data.splitlines():
|
||||
if line == '':
|
||||
continue
|
||||
if line[0] == '#':
|
||||
addcomment ( line[1:] )
|
||||
continue
|
||||
hostlist = re.split ( r'\s+', line )
|
||||
if hostlist[0] != config['blacklists'][source]['hostskey']:
|
||||
# not a matching key
|
||||
continue
|
||||
for host in hostlist[1:]:
|
||||
recordcount += 1
|
||||
addhost ( host )
|
||||
elif config['blacklists'][source]['format'] == 'raw':
|
||||
# comments start "#"
|
||||
for line in data.splitlines():
|
||||
if line == '':
|
||||
continue
|
||||
if line[0] == '#':
|
||||
addcomment ( line[1:] )
|
||||
continue
|
||||
host = line.strip()
|
||||
recordcount += 1
|
||||
addhost ( host )
|
||||
else:
|
||||
sys.exit ( "Unknown format %s for %s" % (config['blacklists'][source]['format'],source) )
|
||||
if recordcount == 0:
|
||||
sys.exit ( "Got recordcount of %d for %s" % (recordcount,source) )
|
||||
|
||||
# if we have a local blacklist, add that also
|
||||
if 'localblacklist' in config:
|
||||
outputdata += "\n%s=============================================================================\n" % commentstart
|
||||
outputdata += "%s Source: Local blacklist from %s\n" % (commentstart,configfile)
|
||||
outputdata += "%s=============================================================================\n\n" % commentstart
|
||||
for host in config['localblacklist']:
|
||||
addhost ( host )
|
||||
|
||||
|
||||
# write the config['hostsfile'] file
|
||||
with open ( config['hostsfile']+'.TMP', 'wt' ) as f:
|
||||
f.write ( outputdata )
|
||||
os.rename ( config['hostsfile'], config['hostsfile']+'.old' )
|
||||
os.rename ( config['hostsfile']+'.TMP', config['hostsfile'] )
|
||||
|
||||
|
||||
# ensure we have a dnsmasq config file - we assume if it's there it's sufficient TODO maybe we should check
|
||||
if not os.path.isfile ( config['dnsmasqblackholeconfig'] ):
|
||||
with open ( config['dnsmasqblackholeconfig']+'.TMP', 'wt' ) as f:
|
||||
f.write ( "addn-hosts=%s\n" % output )
|
||||
os.rename ( config['dnsmasqblackholeconfig']+'.TMP', config['dnsmasqblackholeconfig'] )
|
||||
# TODO reload dnsmasq (SIGHUP re-reads files, but not config)
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
---
|
||||
#cachedir: /var/local/py-hole
|
||||
#cacheprefix: cache-
|
||||
#cacheexpire: 14400 # 4 hours
|
||||
#defaultresponse: 0.0.0.0
|
||||
|
||||
hostsfile: /etc/local-hosts-blackhole
|
||||
dnsmasqblackholeconfig: /etc/dnsmasq.d/local-hosts-blackhole
|
||||
|
||||
|
||||
# see https://github.com/pi-hole/pi-hole/blob/master/adlists.default
|
||||
# Note: the moment we specify blacklists, the base key completely replaces defaults
|
||||
blacklists:
|
||||
StevenBlack:
|
||||
url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
|
||||
format: hosts
|
||||
hostskey: 0.0.0.0
|
||||
malwaredomains: { url: 'https://mirror1.malwaredomains.com/files/justdomains', format: raw }
|
||||
cameleon: { 'url':'http://sysctl.org/cameleon/hosts', 'format':'hosts', 'hostskey':'127.0.0.1' }
|
||||
abuse.ch: { 'url':'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist', 'format':'raw' }
|
||||
disconnect.me_tracking: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt', 'format':'raw' }
|
||||
disconnect.me_ad: { 'url':'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt', 'format':'raw' }
|
||||
# hosts-file.net: { 'url':'https://hosts-file.net/ad_servers.txt', 'format':'hosts0000' },
|
||||
# Windows 10 telemetry: {
|
||||
securemecca.com: { url: 'http://securemecca.com/Downloads/hosts.txt', format: hosts, hostskey: 127.0.0.1 }
|
||||
# currently we support formats of:
|
||||
# * raw
|
||||
# - considers lines starting "#" as comments
|
||||
# - one hostname per line
|
||||
# * hosts
|
||||
# - considers lines starting "#" as comments
|
||||
# - requires "hostskey" matching the IP at the start of the line (anything else ignored)
|
||||
# - multiple hosts per line (typical hosts file with aliases)
|
||||
|
||||
exclusions:
|
||||
www.googleadservices.com: True # needed for google shopping
|
||||
pagead.l.doubleclick.net: True # CNAME for www.googleadservices.com needed for google shopping
|
||||
# Note that "localhost" is always excluded t prevent conflicts
|
||||
|
||||
# we can also add our own local backlist
|
||||
#localblacklist:
|
||||
# - evilhost.example.com # going there does evil stuff
|
||||
|
||||
136
rpzhole
Executable file
136
rpzhole
Executable file
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import logging
|
||||
import yaml
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import argparse
|
||||
import subprocess
|
||||
|
||||
__version__ = '0.1'
|
||||
|
||||
_logger = logging.getLogger("rpzhole")
|
||||
|
||||
def download_blacklist(url, cache_file):
|
||||
"""Downloads and stores a blacklist, returns a filepath"""
|
||||
|
||||
# If we have an existing file, do a header call, check the size and if its the same don't re-download
|
||||
if os.path.exists(cache_file):
|
||||
resp = requests.head(url)
|
||||
if resp.ok and 'content-length' in resp.headers and os.path.getsize(cache_file) == int(resp.headers['content-length']):
|
||||
return True
|
||||
|
||||
# If we don't have the file or missed the cache, download the file
|
||||
resp = requests.get(url, headers={'User-Agent': 'rpzhole %s' % __version__})
|
||||
if resp.ok:
|
||||
with file(cache_file, 'wb') as fobj:
|
||||
for block in resp.iter_content(1024):
|
||||
if block: fobj.write(block)
|
||||
return True
|
||||
|
||||
def parse_blacklist(format, filename=None, fobj=None):
|
||||
if not format in ('hosts', 'raw'):
|
||||
raise Exception('Unsupported format %s' % format)
|
||||
if not fobj:
|
||||
with open(filename, 'rb') as fobj:
|
||||
return parse_blacklist(format, fobj=fobj)
|
||||
else:
|
||||
data = []
|
||||
if format == 'hosts':
|
||||
for line in fobj:
|
||||
if line == '' or line[0] == '#': continue
|
||||
items = re.split ( r'\s+', line )
|
||||
data.extend(items[1:])
|
||||
elif format == 'raw':
|
||||
for line in fobj:
|
||||
if line == '' or line[0] == '#': continue
|
||||
data.append(line.strip())
|
||||
return data
|
||||
|
||||
def write_rpz(hosts, filename, origin='rpz.black.hole'):
|
||||
rpz_header = ["$TTL 60", "$ORIGIN %s" % origin, "@ SOA nonexistent.nodomain.none. dummy.nodomain.none. %d 12h 15m 3w 2h" % time.time(), "NS nonexistant.nodomain.none"]
|
||||
with open(filename, 'wb') as fobj:
|
||||
for line in rpz_header:
|
||||
fobj.write('%s\n' % line)
|
||||
fobj.write('\n')
|
||||
for host in hosts:
|
||||
if host:
|
||||
fobj.write("%s\t\tA\t127.0.0.1\n" % host)
|
||||
|
||||
def main():
|
||||
|
||||
# Parse command line arguments
|
||||
parser = argparse.ArgumentParser(description='Generate Bind9 RPZ zone files from Blacklists')
|
||||
parser.add_argument('--config', dest='config_file')
|
||||
parser.add_argument('--debug', help='Enables logging of debug messages', default=False, action='store_true')
|
||||
parser.add_argument('--silent', help='Disables all logging except for errors, useful for cron execution', default=False, action='store_true')
|
||||
parser.add_argument('--reloadzone', help='Ask Bind to reload the RPZ zone file once completed', default=False, action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Enable verbosity
|
||||
if args.silent:
|
||||
level = logging.ERROR
|
||||
elif args.debug:
|
||||
level = logging.DEBUG
|
||||
else:
|
||||
level = logging.INFO
|
||||
logging.basicConfig(level=level)
|
||||
logging.getLogger("requests").setLevel(logging.WARNING)
|
||||
_logger.debug('Parsed Args: %s', str(args))
|
||||
|
||||
# Load configuration
|
||||
if not args.config_file:
|
||||
default_paths = [os.path.join(os.getcwd(), 'rpzhole.yaml'), os.path.expanduser('~/.rpzhole.yaml'), '/etc/bind/rpzhole.yaml', '/etc/rpzhole.yaml']
|
||||
for filepath in default_paths:
|
||||
if os.path.exists(filepath):
|
||||
_logger.debug('Found configuration file %s', filepath)
|
||||
args.config_file = filepath
|
||||
break
|
||||
else:
|
||||
_logger.error("Unable to find usable configuration file in any of the following locations: %s", ', '.join(default_paths))
|
||||
sys.exit(1)
|
||||
config_file_path = os.path.abspath(os.path.expanduser(args.config_file))
|
||||
_logger.debug('Loading configuration file %s', config_file_path)
|
||||
with open(config_file_path, 'rb') as fobj:
|
||||
config = yaml.load(fobj)
|
||||
|
||||
# Pre-create the cache locations
|
||||
if not os.path.exists(config['cache_dir']):
|
||||
try:
|
||||
os.makedirs(config['cache_dir'])
|
||||
except OSError as e:
|
||||
_logger.error('Unable to create caching directory: %s, exiting...', e)
|
||||
sys.exit(1)
|
||||
|
||||
# Download blacklists
|
||||
blacklist_hosts = []
|
||||
for name, val in config['blacklists'].items():
|
||||
_logger.info('Processing %s blacklist', name)
|
||||
cache_file = os.path.join(config['cache_dir'], config['cache_prefix'] + name + '.list')
|
||||
try:
|
||||
if download_blacklist(val['url'], cache_file):
|
||||
_logger.debug('Parsing %s as format %s', name, val['format'])
|
||||
list_hosts = parse_blacklist(val['format'], cache_file)
|
||||
_logger.info('Adding %d hosts from %s blacklist', len(list_hosts), name)
|
||||
blacklist_hosts.extend(list_hosts)
|
||||
except Exception as e:
|
||||
_logger.error('Unable to download or parse %s blacklist: %s', name, e)
|
||||
|
||||
# Remove duplicates and exclude any hosts on the exclusion list
|
||||
output_hostlist = set(blacklist_hosts) - set(config['exclusions'])
|
||||
_logger.info('%d unique hosts used to create RPZ, %d entries from blacklists, %d exclusion hosts', len(output_hostlist), len(blacklist_hosts), len(config['exclusions']))
|
||||
|
||||
# write RPZ
|
||||
write_rpz(output_hostlist, config['output_filename'], config['origin'])
|
||||
_logger.info('RPZ zone wrote to %s', config['output_filename'])
|
||||
|
||||
# reload RPZ
|
||||
if args.reloadzone:
|
||||
_logger.info('Informing Bind to reload the zone')
|
||||
res = subprocess.call('rndc reload %s' % config['origin'])
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
34
rpzhole.yaml
Normal file
34
rpzhole.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
---
|
||||
|
||||
blacklists:
|
||||
StevenBlack:
|
||||
url: https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
|
||||
format: hosts
|
||||
malwaredomains:
|
||||
url: 'https://mirror1.malwaredomains.com/files/justdomains'
|
||||
format: raw
|
||||
cameleon:
|
||||
url: http://sysctl.org/cameleon/hosts
|
||||
format: hosts
|
||||
abuse.ch:
|
||||
url: https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
|
||||
format: raw
|
||||
disconnect.me_tracking:
|
||||
url: https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
|
||||
format: raw
|
||||
disconnect.me_ad:
|
||||
url: https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
|
||||
format: raw
|
||||
securemecca.com:
|
||||
url: http://securemecca.com/Downloads/hosts.txt
|
||||
format: hosts
|
||||
|
||||
exclusions:
|
||||
- www.googleadservices.com
|
||||
- pagead.l.doubleclick.net
|
||||
|
||||
origin: rpz.home.tensixtyone.com
|
||||
output_filename: ./rpz.zone
|
||||
cache_dir: ./cache/
|
||||
cache_prefix: rpzhole-
|
||||
|
||||
Reference in New Issue
Block a user