Socks5 Support added. Added new proxy site as datasource.

Using Selenium + PhantomJS instead.

Author: ljm625

async_proxy.py

@@ -32,7 +32,7 @@ async def handle_client(reader,writer):
             use_proxy = True
         if use_proxy:
             try:
-                if proxy[2]=='socks':
+                if proxy[2]=='SOCKS5':
                     socks5_addr = aiosocks.Socks5Addr(proxy[0],proxy[1])
                     remote_reader, remote_writer = await aiosocks.open_connection(
                         proxy=socks5_addr, dst=[dest[0],dest[1]], remote_resolve=True)

components/proxy_fetcher.py

@@ -3,10 +3,14 @@
 import yaml
 from bs4 import BeautifulSoup
 from profile.proxy_rules import get_data_on_profile
+from selenium import webdriver
+
 
 class ProxyFetcher(object):
     instance = None
     proxys=None
+    driver = webdriver.PhantomJS()
+
     def __init__(self,urls,candidate_num=5):
         self.urls=urls
         self.candidate=candidate_num
@@ -32,20 +36,28 @@ def yaml_loader(file):
     def proxy_fetcher(self,url):
         resp=requests.get(url)
         if resp.status_code<300:
-            return resp.content
+            self.driver.get(url)
+            return self.driver
         else:
             raise Exception("ERROR : The Proxy Website is DOWN.")
 
     def proxy_parser(self):
+
         try:
             self.proxys=[]
             for server in self.urls:
-                data=self.proxy_fetcher(server['url'])
-                soup = BeautifulSoup(data,"html5lib")
+                # data=self.proxy_fetcher(server['url'])
+                # soup = BeautifulSoup(data,"html5lib")
+                # try:
+                #     self.proxys.extend(get_data_on_profile(server['name'],soup,self.candidate))
+                # except Exception as e:
+                #     pass
+                driver = self.proxy_fetcher(server['url'])
                 try:
-                    self.proxys.extend(get_data_on_profile(server['name'],soup,self.candidate))
+                    self.proxys.extend(get_data_on_profile(server['name'],driver,self.candidate))
                 except Exception as e:
                     pass
+
             print("Got Proxies:")
             print((self.proxys))
         except Exception as e:

profile/proxy_rules.py

@@ -1,29 +1,63 @@
 # Define proxy Profile here.
 
-def get_data_on_profile(profile,soup,number):
+# def get_data_on_profile(profile,soup,number):
+#     # Define profile here
+#     # Return a List with ip and port, example: [["127.0.0.1","1080","http"]]
+#     result=[]
+#
+#     if profile=='cnproxy':
+#         proxy_list = soup.select("tbody tr")
+#         for i in range(0, number):
+#             try:
+#                 result.append([proxy_list[i].select("td:nth-of-type(1)")[0].text.strip("-").strip(),
+#                                proxy_list[i].select("td:nth-of-type(2)")[0].text.strip(), 'http'])
+#             except Exception as e:
+#                 pass
+#     elif profile=='proxynova':
+#         proxy_list = soup.select("tbody tr")
+#         for i in range(0, number):
+#             try:
+#                 result.append( [proxy_list[i].select("td:nth-of-type(1)")[0].abbr.attrs['title'].strip(),
+#                                 proxy_list[i].select("td:nth-of-type(2)")[0].text.strip(),'http'])
+#             except:
+#                 pass
+#     elif profile=='spysone':
+#
+#         proxy_list = soup.select("tr.spy1x,tr.spy1xx")
+#         for i in range(0,number):
+#             proxy_list[i].select("td:nth-of-type(1)")
+#         print(soup)
+#     return result
+
+
+def get_data_on_profile(profile,driver,number):
     # Define profile here
     # Return a List with ip and port, example: [["127.0.0.1","1080","http"]]
     result=[]
 
     if profile=='cnproxy':
-        proxy_list = soup.select("tbody tr")
+        proxy_list = driver.find_elements_by_css_selector("tbody tr")
         for i in range(0, number):
             try:
-                result.append([proxy_list[i].select("td:nth-of-type(1)")[0].text.strip("-").strip(),
-                               proxy_list[i].select("td:nth-of-type(2)")[0].text.strip(), 'http'])
+                result.append([proxy_list[i].find_elements_by_css_selector("td:nth-of-type(1)")[0].text.strip("-").strip(),
+                               proxy_list[i].find_elements_by_css_selector("td:nth-of-type(2)")[0].text.strip(), 'HTTP'])
             except Exception as e:
                 pass
     elif profile=='proxynova':
-        proxy_list = soup.select("tbody tr")
+        proxy_list = driver.find_elements_by_css_selector("tbody tr")
         for i in range(0, number):
             try:
-                result.append( [proxy_list[i].select("td:nth-of-type(1)")[0].abbr.attrs['title'].strip(),
-                                proxy_list[i].select("td:nth-of-type(2)")[0].text.strip(),'http'])
+                result.append( [proxy_list[i].find_elements_by_css_selector("td:nth-of-type(1)")[0].text.strip(),
+                                proxy_list[i].find_elements_by_css_selector("td:nth-of-type(2)")[0].text.strip(),'HTTP'])
             except:
                 pass
     elif profile=='spysone':
-        proxy_list = soup.select("tr.spy1x,tr.spy1xx")
-        for i in range(0,number):
-            proxy_list[i].select("td:nth-of-type(1)")
-        print(soup)
+
+        proxy_list = driver.find_elements_by_css_selector("tr.spy1x,tr.spy1xx")
+        if len(proxy_list)<number+2:
+            number = len(proxy_list)-2
+        for i in range(2,number+2):
+            proxy_info = proxy_list[i].text.strip().split(" ")
+            result.append([proxy_info[1].split(":")[0],proxy_info[1].split(":")[1],proxy_info[2]])
+        print(driver)
     return result

requirements.txt

@@ -6,4 +6,5 @@ html5lib
 dnslib
 pysocks
 requests[socks]
-aiosocks
+aiosocks
+selenium