From 1fc733896b80b725e2df909d6c74c249435251ef Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 11 Mar 2022 15:38:09 +0700 Subject: [PATCH] action.py : Add duplicat module to production that'll help remove duplicate ruleset in another blocklist version --- action.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/action.py b/action.py index 0288f84..9a74680 100644 --- a/action.py +++ b/action.py @@ -4,6 +4,7 @@ sys.path.append('/filters-maker') import os import crawler import maker +import duplicat incoming = "/reprwiki/Private-build/ucate/domains.txt" excluded = "/repros/Resources/excluded.txt" @@ -43,6 +44,7 @@ crawler.IP_URL_FILTERING(incoming) crawler.excluded(excluded, incoming) crawler.blankremover(incoming) crawler.sort(incoming) +duplicat.add_file(incoming, "/reprwiki/Private-build/ruleset.txt") UCATEline = maker.linecounter(incoming) maker.RPZBlocklist(excluded, incoming, rpz_locat ,Version) maker.HOSTBlocklist(excluded, incoming, hosts_locat ,Version) @@ -79,6 +81,8 @@ crawler.IP_URL_FILTERING(incoming) crawler.excluded(excluded, incoming) crawler.blankremover(incoming) crawler.sort(incoming) +duplicat.check_n_kill_dupes("/reprwiki/Private-build/ruleset.txt", incoming) +duplicat.add_file(incoming, "/reprwiki/Private-build/ruleset.txt") VENETOline = maker.linecounter(incoming) maker.RPZBlocklist(excluded, incoming, rpz_locat ,Version) maker.HOSTBlocklist(excluded, incoming, hosts_locat ,Version)