Duplicate line issues might fixed

excluder.py

@@ -22,16 +22,16 @@ def add_file(incoming,excluded_in):
     data += data2
     with open (incoming, 'w') as fp:
         fp.write(data + '\n')
-    with open(incoming, 'r') as f:
-        lines = set(f.readlines())
-    with open(incoming, 'w') as f:
-          f.writelines(set(lines))
     with open(incoming ,'r') as f:
         lines = f.read().split()
     with open(incoming ,'w') as f:
         for line in lines:
             if line.strip() and not line.startswith((tuple(comment_roc))):
                 f.write('\n'.join([line + '\n']))
+    with open(incoming, 'r') as f:
+        lines = set(f.readlines())
+    with open(incoming, 'w') as f:
+          f.writelines(set(lines))
     crawler.sort(incoming)
     os.remove(excluded_in)