From 9f3e40a47496830a50e74dad29be76e669d9b87e Mon Sep 17 00:00:00 2001 From: Kale Date: Mon, 2 Aug 2021 08:30:49 -0700 Subject: [PATCH] Update 'crawler.py' --- crawler.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crawler.py b/crawler.py index bd174c6..852da2e 100644 --- a/crawler.py +++ b/crawler.py @@ -68,6 +68,20 @@ def killingdup(duplicated_file): print("++ successful!") f.close() +def excluded(excluded_file ,incoming): + with open(excluded ,'r') as f: + exclude = f.read().split() + with open(incoming ,'r') as f: + lines = f.read().splitlines() # read lines + with open(output ,'w') as f: + for line in lines: + if line.strip() and not line in exclude and not line.startswith(';'): + f.write('\n'.join([line + ' CNAME .\n'])) + elif line.startswith((';','$','@',' IN')): + f.write('\n'.join([line + '\n'])) + elif not line.strip(): + f.write('\n'.join([line + '\n'])) + if __name__ == "__main__": download_filters('https://filters.kylz.nl/RPZ/someonewhocares/rpz.txt','test.txt') filtering('test.txt')