From 160c116ff117e25e249d3c0d66b9624c425ceb56 Mon Sep 17 00:00:00 2001 From: minoplhy Date: Wed, 27 Apr 2022 23:04:01 +0700 Subject: [PATCH] crawler : additional filtering --- crawler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crawler.py b/crawler.py index 4e6c723..ff6cb47 100644 --- a/crawler.py +++ b/crawler.py @@ -134,6 +134,15 @@ def filteringcon(filters_regex_one): print('dnsmasq filtering Operation Completed!') f.close() + with open(filters_regex_one) as f: + file = f.read().split('\n') + for i in range(len(file)): + file[i] = re.sub('^\.', '', file[i]) + with open(filters_regex_one, 'w') as f1: + f1.writelines(["%s\n" % item for item in file]) + print('additional filtering Operation Completed!') + f.close() + remove_words = ['localhost','localhost.localdomain','local','broadcasthost','loopback','ip6-localnet','ip6-mcastprefix','ip6-allnodes','ip6-allrouters','ip6-allhosts','ip6-loopback',' CNAME rpz-passthru.'] with open(filters_regex_one, 'r') as f: