diff --git a/filters.sh b/filters.sh index 64ff9a7..0839bca 100644 --- a/filters.sh +++ b/filters.sh @@ -5,6 +5,7 @@ mkdir /repros/$INPUT_DESTINATION_FOLDER/$INPUT_DESTINATION_VERSION pip3 install -r /filters-maker/requirements.txt python3 /filters-maker/crawler.py /repros/$INPUT_DESTINATION_FOLDER/$INPUT_DESTINATION_VERSION/domains.txt python3 /filters-maker/maker-rpz.py /repros/Resources/excluded.txt /repros/$INPUT_DESTINATION_FOLDER/$INPUT_DESTINATION_VERSION/domains.txt /repros/$INPUT_DESTINATION_FOLDER/$INPUT_DESTINATION_VERSION/rpz.txt +python3 /filters-maker/maker-domains.py /repros/Resources/excluded.txt /repros/$INPUT_DESTINATION_FOLDER/$INPUT_DESTINATION_VERSION/domains.txt cd /repros git config --local user.name $INPUT_GIT_NAME git config --local user.email $INPUT_GIT_EMAIL diff --git a/maker-domains.py b/maker-domains.py new file mode 100644 index 0000000..75e85ba --- /dev/null +++ b/maker-domains.py @@ -0,0 +1,41 @@ +import os +import sys +import datetime +import pytz + + +excluded = sys.argv[1] +input = sys.argv[2] +UTC = pytz.utc +date = datetime.datetime.now(UTC) + +def linecounter(): + with open(input) as f: + for i, l in enumerate(f): + pass + return i + 1 + +def domainsbuilding(): + with open(excluded ,'r') as f: + exclude = f.read().split() + with open(input ,'r') as f: + lines = f.read().splitlines() # read lines + with open(input ,'w') as f: + f.write('# Title : Minoplhy Personal Blocklist\n') + f.write('# Description : My Very Personal DNS Blocklist plus crawling from the source\n') + f.write('# Source : Source.txt\n') + f.write('# Rule Counter : ' + str(linecounter()) +' Rules\n') + f.write('# Format : Domains\n') + f.write('# Licenses : MIT\n') + f.write('# Compiled Date : ' + str(date) +'\n\n') + for line in lines: + if line.strip() and not line in exclude and not line.startswith('#'): + f.write('\n'.join([line + '\n'])) + if line.startswith((';','$','@',' IN')): + f.write('\n'.join([line + '\n'])) + if not line.strip(): + f.write('\n'.join([line + '\n'])) + f.close() + +domainsbuilding() +exit() \ No newline at end of file