Part2_features_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hoJx7100DurI"
   },
   "source": [
    "# Part II: Feature extarction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "9mPg88DKCAtE"
   },
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlparse\n",
    "from urllib.parse import urlparse,urlencode\n",
    "import ipaddress\n",
    "import re\n",
    "import socket\n",
    "import requests\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Nh8SCpXpD2d8"
   },
   "source": [
    "## II.1. Features based on Adress"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are some obvious features related to URL format and other suspicious forms that can be included by attackers to propagate their threats. In this part we code some functions that calculate those attributes. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OuNZ4IkFEA51"
   },
   "source": [
    "**II.1.1 Domain of the URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "ZA79WOxsD6zI"
   },
   "outputs": [],
   "source": [
    "def getDomain(url):  \n",
    "    domain = urlparse(url).netloc\n",
    "    if re.match(r\"^www.\",domain):\n",
    "        domain = domain.replace(\"www.\",\"\")\n",
    "        return(domain)\n",
    "    return(\"NULL\") \n",
    "# if we can't extract the domain du to anomaly in the link we return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8bKQrGn7ESBN"
   },
   "source": [
    " **II.1.2 IP Address in the URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "Jx97rbaqEU72"
   },
   "outputs": [],
   "source": [
    "def getIPAddress(url):\n",
    "    try:\n",
    "        address=socket.gethostbyname(getDomain(url))\n",
    "        return(address)\n",
    "    except:\n",
    "        return(None)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-c5YGLIuEeFJ"
   },
   "source": [
    "**II.1.3 \"@\" Symbol in URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "h6urnvwEEZMd"
   },
   "outputs": [],
   "source": [
    "def getExitanceSymbol(url):\n",
    "    if '@' in url:\n",
    "        return(1)\n",
    "    else:\n",
    "        return(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ipKffsdwEk2r"
   },
   "source": [
    "**II.1.4 Length of URL** \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "hP6t7FBwEkKi"
   },
   "outputs": [],
   "source": [
    "def LengthUrl(url):\n",
    "    return(len(url))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AI371oCCElW9"
   },
   "source": [
    "**II.1.5 Length of Domain**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "dr0dLh2BElf-"
   },
   "outputs": [],
   "source": [
    "def LengthDomain(url):\n",
    "    return(len(getDomain(url)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dnUsY1vMEloN"
   },
   "source": [
    "**II.1.6 Depth of Domain**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "FYtD1r8bElvX"
   },
   "outputs": [],
   "source": [
    "def getDepth(url):\n",
    "    ch=getDomain(url)\n",
    "    l=ch.split('/')\n",
    "    depth=0\n",
    "    for i in l:\n",
    "        if len(i)>0:\n",
    "            depth=depth+1\n",
    "    return(depth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y3nuTGCiEl1H"
   },
   "source": [
    "**II.1.7 Number of \"-\" in URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "PEaRN9I9El6L"
   },
   "outputs": [],
   "source": [
    "def occurence_tirets(url):\n",
    "    l=url.split('-')\n",
    "    nbr=0\n",
    "    for i in l:\n",
    "        if len(i)>0:\n",
    "            nbr=nbr+1\n",
    "    return(nbr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YuZFRAP_El_y"
   },
   "source": [
    "**II.1.8 Redirection \"//\" in URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "Zia7-i4wEmEs"
   },
   "outputs": [],
   "source": [
    "#  redirection to other website\n",
    "def getPositionRedirection(url):\n",
    "    l=[i for i in url]\n",
    "    for i in range (len(l)-1):\n",
    "        if l[i]==\"/\" and l[i+1]==\"/\":\n",
    "            return(i)\n",
    "    return(None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FpHi2Ja1EmJj"
   },
   "source": [
    "**II.1.9 Number of % in URL**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "uaCuCVQaEmOH"
   },
   "outputs": [],
   "source": [
    "# nombre de %\n",
    "def getNumberPourcentage(url):\n",
    "    l=url.split('%')\n",
    "    nbr=0\n",
    "    for i in l:\n",
    "        if len(i)>0:\n",
    "            nbr=nbr+1\n",
    "    return(nbr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f-tCRFknEmTL"
   },
   "source": [
    "**II.1.10 \"https\" in Domain name**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "uE_w93w1EmXY"
   },
   "outputs": [],
   "source": [
    "#Detect the string https in domain\n",
    "\n",
    "def httpInDomain(url):\n",
    "  domain = urlparse(url).netloc\n",
    "  if 'https' in domain:\n",
    "    return 1\n",
    "  else:\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FgVP_yQfFeMN"
   },
   "source": [
    "**II.1.11 Using URL Shortening Services “TinyURL”**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "Wk4bLA2EFeSr"
   },
   "outputs": [],
   "source": [
    "#Tiny urls:\n",
    "shortening_services = r\"bit\\.ly|goo\\.gl|shorte\\.st|go2l\\.ink|x\\.co|ow\\.ly|t\\.co|tinyurl|tr\\.im|is\\.gd|cli\\.gs|\" \\\n",
    "                      r\"yfrog\\.com|migre\\.me|ff\\.im|tiny\\.cc|url4\\.eu|twit\\.ac|su\\.pr|twurl\\.nl|snipurl\\.com|\" \\\n",
    "                      r\"short\\.to|BudURL\\.com|ping\\.fm|post\\.ly|Just\\.as|bkite\\.com|snipr\\.com|fic\\.kr|loopt\\.us|\" \\\n",
    "                      r\"doiop\\.com|short\\.ie|kl\\.am|wp\\.me|rubyurl\\.com|om\\.ly|to\\.ly|bit\\.do|t\\.co|lnkd\\.in|db\\.tt|\" \\\n",
    "                      r\"qr\\.ae|adf\\.ly|goo\\.gl|bitly\\.com|cur\\.lv|tinyurl\\.com|ow\\.ly|bit\\.ly|ity\\.im|q\\.gs|is\\.gd|\" \\\n",
    "                      r\"po\\.st|bc\\.vc|twitthis\\.com|u\\.to|j\\.mp|buzurl\\.com|cutt\\.us|u\\.bb|yourls\\.org|x\\.co|\" \\\n",
    "                      r\"prettylinkpro\\.com|scrnch\\.me|filoops\\.info|vzturl\\.com|qr\\.net|1url\\.com|tweez\\.me|v\\.gd|\" \\\n",
    "                      r\"tr\\.im|link\\.zip\\.net\"\n",
    "\n",
    "def tinyURL(url):\n",
    "    match=re.search(shortening_services,url)\n",
    "    if match:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9IHYPAQzFtPi"
   },
   "source": [
    "## II.2.HTML & JS based Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section, we will extract information based on the HTML and JavaScript features of websites."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "K0yIN5KNFeXw"
   },
   "source": [
    "**II.2.1 IFrame** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "H5eOC2eTF135"
   },
   "outputs": [],
   "source": [
    " import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "BzUhP1KuFec6"
   },
   "outputs": [],
   "source": [
    "def iframe(url):\n",
    "    alarm = threading.Timer(delay_time, watchdog) # works only on linux , used to add timeout\n",
    "    alarm.start()\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "    except:\n",
    "        response = \"\"\n",
    "    if response == \"\":\n",
    "        return 1\n",
    "    else:\n",
    "        if re.findall(r\"[|]\", response.text):\n",
    "            return 0\n",
    "        else:\n",
    "            return 1\n",
    "    alarm.cancel()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ChgidBISFehd"
   },
   "source": [
    "**II.2.2 Status Bar Customization:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "pODttzT3FemV"
   },
   "outputs": [],
   "source": [
    "def mouseOver(url): \n",
    "    alarm = threading.Timer(delay_time, watchdog)\n",
    "    alarm.start()\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "    \n",
    "        if response == \"\" :\n",
    "            return 1\n",
    "        else:\n",
    "            if re.findall(\"\", response.text):\n",
    "              return 1\n",
    "            else:\n",
    "              return 0\n",
    "    except:\n",
    "            return(None)\n",
    "    alarm.cancel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**II.2.3 Number of Redirection:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NumberRedirects(url):\n",
    "    response = requests.get(url, headers={'User-Agent': 'Google Chrome'})\n",
    "    if response.history:\n",
    "        return 1 + NumberRedirects(response.url)        \n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yIfa2o1jFez4"
   },
   "source": [
    "## II.3. Abnormal based features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n9MyX5VwDYqF"
   },
   "source": [
    "**II.3.1 URL of Anchor:**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def URLofAnchor(url):\n",
    "    domain = urlparse(url).netloc \n",
    "\n",
    "    soup = BeautifulSoup(requests.get(url).content, \"html.parser\").find_all(\"a\")\n",
    "    total = len(soup)\n",
    "    count = 0\n",
    "    for ele in soup[:]:\n",
    "        ele = str(ele)\n",
    "        #print(ele)\n",
    "        if 'href=\"https://' in ele and domain not in ele:\n",
    "            count+=1\n",
    "        if 'href=\"#\"' in ele or 'href=\"#content\"' in ele or 'href=\"#skip\"' in ele or 'href=\"JavaScript ::void(0)\"' in ele:\n",
    "            count+=1\n",
    "\n",
    "    if total == 0:\n",
    "        return None\n",
    "    return count/total*100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ueFJoW66HYUH"
   },
   "source": [
    "# II.4. Domain based Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qXTBlbL6Dg9B"
   },
   "source": [
    "**II.4.1 Number of Months Alive:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import whois\n",
    "from dateutil import relativedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def checkWHOIS_exists_months(url):\n",
    "    #url = re.sub(r'(?:https?://)([^/]+)(?:\\S+)', r\"\\1\", url)\n",
    "    w = whois.whois(url)\n",
    "    #print(w)\n",
    "    if w[\"creation_date\"] == None:\n",
    "        print(\"test\")\n",
    "        return None\n",
    "    else:\n",
    "        date_crea = w[\"creation_date\"]\n",
    "        try:\n",
    "            if len(date_crea) > 1:\n",
    "                date_crea = date_crea[-1]\n",
    "        except:\n",
    "            pass\n",
    "        diff = relativedelta.relativedelta(date.today(),date_crea)\n",
    "        print(diff)\n",
    "        return diff.months + diff.years * 12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "112QasYXHjOM"
   },
   "source": [
    "**II.4.2 isinAlexa:**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def isinAlexa(url):\n",
    "    #url = re.sub(r'(?:https?://)([^/]+)(?:\\S+)', r\"\\1\", url)\n",
    "    url = url[12:]\n",
    "    print(url)\n",
    "    with open('top-1m.csv', 'rt') as f:\n",
    "        reader = csv.reader(f, delimiter=',') # good point by @paco\n",
    "        for row in reader:\n",
    "            if row[1] == url:\n",
    "                return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "j9al0UC1JBQ6"
   },
   "source": [
    "**II.4.3 Number of Links Pointing to Page:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NumbLinkToDomain(url):\n",
    "    domain = re.sub(r'(?:https?://)([^/]+)(?:\\S+)', r\"\\1\", url)\n",
    "    reqs = requests.get(url)\n",
    "    soup = BeautifulSoup(reqs.text, 'html.parser')\n",
    "\n",
    "    yes=0\n",
    "\n",
    "    for link in soup.find_all('a'):\n",
    "        link = link.get('href')\n",
    "        if domain in link:\n",
    "            yes+=1\n",
    "    return yes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X-Z3izA5Irt1"
   },
   "source": [
    "**II.4.4 Indexed by Google:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexbyGoogle(url):\n",
    "    google = \"https://www.google.com/search?q=site:\" + url + \"&hl=en\"\n",
    "    response = requests.get(google, cookies={\"CONSENT\": \"YES+1\"})\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    not_indexed = re.compile(\"did not match any documents\")\n",
    "\n",
    "    if soup(text=not_indexed):\n",
    "        return 0\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II.5 URL Features computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def featureExtractions(url,label):\n",
    "\n",
    "    features = []\n",
    "    \n",
    "    features.append(url)\n",
    "    features.append(URLofAnchor(url))\n",
    "    features.append(NumberRedirects(url))\n",
    "    features.append(checkWHOIS_exists_months(url))\n",
    "    features.append(isinAlexa(url))\n",
    "    features.append(NumbLinkToDomain(url))\n",
    "    features.append(indexbyGoogle(url))\n",
    "    features.append(getDomain(url))\n",
    "    features.append(getIPAddress(url))\n",
    "    features.append(getExitanceSymbol(url))\n",
    "    features.append(LengthUrl(url))\n",
    "    features.append(LengthDomain(url))\n",
    "    features.append(getDepth(url))\n",
    "    features.append(occurence_tirets(url))\n",
    "    features.append(getPositionRedirection(url))\n",
    "    features.append(getNumberPourcentage(url))\n",
    "    features.append(httpInDomain(url))\n",
    "    features.append(tinyURL(url))\n",
    "    features.append(iframe(url))\n",
    "    features.append(mouseOver(url))\n",
    "    features.append(label)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**II.5.1 Phishing Websites**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    https://75e523b0-579a-4d05-aa9e-2d1ccde20fc0.i...\n",
       "1      http://vystar.security.canarotransportes.com.br\n",
       "2    http://vintedsite-market.45444342655.xyz/qiaex...\n",
       "3               https://fwwwfrererre.000webhostapp.com\n",
       "4            https://e-tax.form.3649-indentificamz.com\n",
       "Name: url, dtype: object"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#load phishing websites\n",
    "import pandas as pd\n",
    "df = pd.read_csv('./Datasets/phish_file_brut.csv')\n",
    "df=df['url']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "test\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'threading' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_11984\\2657797677.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# to track the execution\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0murl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeatureExtractions\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_11984\\745452216.py\u001b[0m in \u001b[0;36mfeatureExtractions\u001b[1;34m(url, label)\u001b[0m\n\u001b[0;32m     21\u001b[0m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhttpInDomain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtinyURL\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmouseOver\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[0mfeatures\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_11984\\1002143436.py\u001b[0m in \u001b[0;36miframe\u001b[1;34m(url)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0miframe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0malarm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mthreading\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTimer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdelay_time\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwatchdog\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0malarm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'threading' is not defined"
     ]
    }
   ],
   "source": [
    "features = []\n",
    "label = 1 # label of phish\n",
    "for i in range(0, df.shape[0]):\n",
    "    print(i) # to track the execution\n",
    "    url = df[i] \n",
    "    features.append(featureExtractions(url,label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Link</th>\n",
       "      <th>Domain</th>\n",
       "      <th>Have_IP</th>\n",
       "      <th>Have_@</th>\n",
       "      <th>URL_Length</th>\n",
       "      <th>Domain_Length</th>\n",
       "      <th>URL_Depth</th>\n",
       "      <th>occurence_tirets</th>\n",
       "      <th>Redirection</th>\n",
       "      <th>Number_%</th>\n",
       "      <th>https_Domain</th>\n",
       "      <th>TinyURL</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://shadetreetechnology.com/V4/validation/a...</td>\n",
       "      <td>shadetreetechnology.com</td>\n",
       "      <td>34.102.136.180</td>\n",
       "      <td>0</td>\n",
       "      <td>77</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://support-appleld.com.secureupdate.duila...</td>\n",
       "      <td>support-appleld.com.secureupdate.duilawyeryork...</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>126</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://appleid.apple.com-app.es/</td>\n",
       "      <td>appleid.apple.com-app.es</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.shadetreetechnology.com/V4/validati...</td>\n",
       "      <td>shadetreetechnology.com</td>\n",
       "      <td>34.102.136.180</td>\n",
       "      <td>0</td>\n",
       "      <td>81</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://html.house/l7ceeid6.html</td>\n",
       "      <td>html.house</td>\n",
       "      <td>104.236.175.252</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Link  \\\n",
       "0  http://shadetreetechnology.com/V4/validation/a...   \n",
       "1  https://support-appleld.com.secureupdate.duila...   \n",
       "2                   http://appleid.apple.com-app.es/   \n",
       "3  http://www.shadetreetechnology.com/V4/validati...   \n",
       "4                    http://html.house/l7ceeid6.html   \n",
       "\n",
       "                                              Domain          Have_IP  Have_@  \\\n",
       "0                            shadetreetechnology.com   34.102.136.180       0   \n",
       "1  support-appleld.com.secureupdate.duilawyeryork...             None       0   \n",
       "2                           appleid.apple.com-app.es             None       0   \n",
       "3                            shadetreetechnology.com   34.102.136.180       0   \n",
       "4                                         html.house  104.236.175.252       0   \n",
       "\n",
       "   URL_Length  Domain_Length  URL_Depth  occurence_tirets  Redirection  \\\n",
       "0          77             23          1                 1            5   \n",
       "1         126             50          1                 2            6   \n",
       "2          32             24          1                 2            5   \n",
       "3          81             23          1                 1            5   \n",
       "4          31             10          1                 1            5   \n",
       "\n",
       "   Number_%  https_Domain  TinyURL  label  \n",
       "0         1             0        0      1  \n",
       "1         1             0        0      1  \n",
       "2         1             0        0      1  \n",
       "3         1             0        0      1  \n",
       "4         1             0        0      1  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_names = [\"Link\",\"URLofAnchor\",\"NumberRedirects\",\"checkWHOIS_exists\",\"isinAlexa\",\"NumbLinkToDomain\",\"indexbyGoogle\",\"Domain\",\"Have_IP\",\"Have_@\",\"URL_Length\",\"Domain_Length\",\"URL_Depth\",\"occurence_tirets\",\"Redirection\",\"Number_%\",\"https_Domain\",\"TinyURL\",\"iFrame\",\"MouseOver\",\"label\"]\n",
    "\n",
    "#coverting the list to a dataframe\n",
    "phishing = pd.DataFrame(features, columns= feature_names)\n",
    "phishing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the dataframe in a csv file\n",
    "phishing.to_csv('./Datasets/phish_file.csv', index= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**II.5.2 Ligitimate websites**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load ligitimate websites\n",
    "import pandas as pd\n",
    "df = pd.read_csv('./Datasets/ligit_file_brut.csv')\n",
    "df=df['url']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = []\n",
    "label = 0 # label of phish\n",
    "for i in range(0, df.shape[0]):\n",
    "    print(i) # to track the execution\n",
    "    url = df[i] \n",
    "    features.append(featureExtractions(url,label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = [\"Link\",\"URLofAnchor\",\"NumberRedirects\",\"checkWHOIS_exists\",\"isinAlexa\",\"NumbLinkToDomain\",\"indexbyGoogle\",\"Domain\",\"Have_IP\",\"Have_@\",\"URL_Length\",\"Domain_Length\",\"URL_Depth\",\"occurence_tirets\",\"Redirection\",\"Number_%\",\"https_Domain\",\"TinyURL\",\"iFrame\",\"MouseOver\",\"label\"]\n",
    "\n",
    "\n",
    "#coverting the list to a dataframe\n",
    "phishing = pd.DataFrame(features, columns= feature_names)\n",
    "phishing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save the dataframe in a csv file\n",
    "phishing.to_csv('./Datasets/ligit_file.csv', index= False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}