added blocksite updater function with new PAC file.

Author: ljm625

components/blocksites_updater.py

@@ -1,28 +1,34 @@
 import time
-
+import yaml
 import re
 import requests
 
 
 class BlocksitesUpdater(object):
 
-    regex_rules=r'http.?://([0-9A-Za-z.]*)/'
+    regex_rules=r'http.?://\*?([0-9A-Za-z.\-]*)/'
     def __init__(self,url):
-        self.url=url
-        pass
+        self.url=url.format(time=int(time.time()*100))
+        self.get()
+        self.parse_domain_list()
 
     def get(self):
         resp = requests.get(self.url)
         self.data = [i.strip('"') for i in resp.text.split(',')]
+        return self.data
 
 
 
     def parse_domain_list(self):
         domain = set()
         for data in self.data:
             result=re.search(self.regex_rules, data)
-            domain.add(result.group(1))
-        return domain
+            try:
+                domain.add(result.group(1).strip("."))
+            except:
+                pass
+        self.domain=domain
+
 
 
 

components/pac_generator.py

@@ -4,13 +4,17 @@
 from jinja2 import FileSystemLoader
 import os
 
+from components.blocksites_updater import BlocksitesUpdater
+
+
 class PacGenerator(object):
-    def __init__(self,config):
+    def __init__(self,config,block_list=None):
         self.env = Environment(loader=FileSystemLoader('template'))
         def yaml_loader(file):
             with open(file) as f:
                 return yaml.load(f)
         self.config=yaml_loader(config)
+        self.block_list=block_list
 
 
     def get_ip(self):
@@ -25,15 +29,20 @@ def get_ip(self):
 
     def build_pac(self):
         info={}
-        info['proxy_domain']=self.config['dns_domain']
+        if self.block_list:
+            info['block_urls']=self.block_list
+        else:
+            info['block_urls']=self.config['dns_domain']
+        info['direct_urls']=self.config['direct_urls']
         try:
             info['proxy_ip']=os.environ["IPADDR"]
         except:
             info['proxy_ip']=self.get_ip()
         info['proxy_port']=self.config.get("proxy_port")
-        template = self.env.get_template("proxy.pac")
+        template = self.env.get_template("new_proxy.pac")
         self.file_writer(template.render(info=info))
 
     def file_writer(self,data):
         with open("pac/proxy.pac",'w+') as file:
-            file.write(data)
+            file.write(data)
+

components/proxy_checker.py

@@ -28,7 +28,11 @@ def yaml_loader(file):
         else:
             config=yaml_loader('config.yaml')
             check_url=config.get('validate_url')
-            speedtest_url=config.get('speedtest_url')
+            https_enabled = config.get('https_enabled')
+            if https_enabled:
+                speedtest_url=config.get('speedtest_url_https')
+            else:
+                speedtest_url=config.get('speedtest_url')
             timeout=config.get('timeout')
             checker_timeout=config.get('checker_timeout')
             speedtest_times=config.get("speedtest_times")

config.yaml

@@ -40,10 +40,18 @@ validate_url: "http://ipservice.163.com/isFromMainland"
 speedtest_url : "http://music.163.com/test"
 speedtest_url_https : "https://music.163.com"
 
-https_enabled : true,
+https_enabled : true
 
 # HTTPS Related options
 
+
+# Blocksites updater
+
+enable_blocksites_updater : true
+updater_url : ""
+
+# Blocksites updater
+
 speedtest_times : 3
 timeout: 5
 checker_timeout: 3
@@ -64,4 +72,14 @@ upstream_dns : "8.8.8.8"
 
 # Disable normal HTTP Proxy
 # this is useful if u make a public server on the internet and avoid them to use it as a public HTTP Proxy. (Need to use with PAC or DNS)
-disable_proxy : false
+disable_proxy : false
+
+
+# Auto Generate Infos.
+
+direct_urls :
+- "*inter.iqiyi.com*"
+- "*file.xiami.com*"
+- "*pcvideoaliyun.titan.mgtv.com*"
+- "*vhotlx.video.qq.com*"
+- "*newflv.sohu.ccgslb.net*"

main.py

@@ -11,6 +11,7 @@
 import yaml
 
 from async_proxy import handle_client
+from components.blocksites_updater import BlocksitesUpdater
 from components.pac_generator import PacGenerator
 from components.proxy_checker import ProxyChecker
 from components.proxy_helper import ProxyHelper
@@ -26,8 +27,16 @@ def yaml_loader(file):
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO)
-    PacGenerator("config.yaml").build_pac()
+
     config = yaml_loader("config.yaml")
+    if config.get('enable_blocksites_updater'):
+        block_generator = BlocksitesUpdater(config.get("updater_url"))
+    else:
+        block_generator = None
+
+    PacGenerator("config.yaml",block_generator.data if block_generator else None).build_pac()
+
+
 
     # pid = os.fork()
     # if pid == 0:
@@ -65,6 +74,10 @@ def yaml_loader(file):
             sys.exit(1)
 
     proxy_helper=ProxyHelper.get_instance()
+
+    if block_generator:
+        proxy_helper.black_list=list(block_generator.domain)
+
     helper = Helper("Helper",proxy_helper)
     helper.start()
 

template/new_proxy.pac

@@ -1,16 +1,15 @@
 var block_urls={{info.block_urls}}
 var direct_urls={{info.direct_urls}}
-var proxies_str='{%- for item in info.proxies %}PROXY {{item}};{%- endfor %}';
 function FindProxyForURL(url, host) {
 if (shExpMatch(url,'http://cache.video.qiyi.com/vms*')) return proxies_str;
 for(var i in direct_urls){
  var direct_url= direct_urls[i];
 if (shExpMatch(url,direct_url)) return 'DIRECT';
  }
 for(var i in block_urls){
-var block_url=block_urls[i]
+var block_url=block_urls[i];
  if (shExpMatch(url,block_url)){
-return proxies_str;
+return "PROXY {{info.proxy_ip}}:{{info.proxy_port}};";
  }
 }
   return 'DIRECT';