Classification_pyspark.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "97kc4sXmM9s4",
        "outputId": "cec9cef4-2fc1-4a7e-9326-8d38568632a7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.2.0.tar.gz (281.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.3 MB 45 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.2\n",
            "  Downloading py4j-0.10.9.2-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 78.4 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.2.0-py2.py3-none-any.whl size=281805911 sha256=1996d815de05c1da65e1fa89deba165be91ee7b3b005525a150eb42eebad3526\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/de/d2/9be5d59d7331c6c2a7c1b6d1a4f463ce107332b1ecd4e80718\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.2 pyspark-3.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YDog6161ATRK"
      },
      "outputs": [],
      "source": [
        "from itertools import chain\n",
        "from pyspark.sql.functions import count, mean, when, lit, create_map, regexp_extract\n",
        "from pyspark.sql import Row\n",
        "from pyspark.sql.session import SparkSession\n",
        "from pyspark import SparkContext\n",
        "import pandas as pd\n",
        "from pyspark.mllib.regression import LabeledPoint\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RBvWkPkBzjSR"
      },
      "outputs": [],
      "source": [
        "sc = SparkContext()\n",
        "spark = SparkSession(sc)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**TP simple**"
      ],
      "metadata": {
        "id": "GvRzs0YUUNds"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LYImNP_dApfE"
      },
      "outputs": [],
      "source": [
        "def clean(file):\n",
        "    Liste = []\n",
        "    for ele in file[4:]:\n",
        "        x = ele.strip('\"')\n",
        "        if x == \"?\":\n",
        "            Liste += [float(0)]\n",
        "        else:\n",
        "            Liste += [float(x)]\n",
        "    return Liste\n",
        "\n",
        "df = sc.textFile(\"/train_noheader.tsv\")\n",
        "\n",
        "df_split = df.map(lambda x : x.split(\"\\t\"))\n",
        "\n",
        "df_cleaned = df_split.map(lambda x : LabeledPoint(int(clean(x)[-1]),clean(x)[:-1],))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iiVgqUxA1_N4",
        "outputId": "7ecb44c3-bc7a-4ee2-845d-f71143a17200"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabeledPoint(0.0, [0.789131,2.055555556,0.676470588,0.205882353,0.047058824,0.023529412,0.443783175,0.0,0.0,0.09077381,0.0,0.245831182,0.003883495,1.0,1.0,24.0,0.0,5424.0,170.0,8.0,0.152941176,0.079129575])"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "from pyspark.rdd import RDD\n",
        "df_cleaned.first()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xOf0lwzUW8uV",
        "outputId": "6e04e56b-7887-42a7-c453-3f56774322e3"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7395"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "df_cleaned.count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nmYSOpF-XFQ1"
      },
      "outputs": [],
      "source": [
        "(training , test) = df_cleaned.randomSplit([0.8,0.2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ktNdRDkLXOgQ",
        "outputId": "5849d857-4a80-4552-8074-0fb2e4a69f2f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PythonRDD[10] at RDD at PythonRDD.scala:53"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "training.cache()\n",
        "test.cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vl2oTSosXSuh",
        "outputId": "dccff836-96d1-4d91-8382-3aed380dd545"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.4943899018232819\n"
          ]
        }
      ],
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.05)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_RFLgYFiXlZW",
        "outputId": "4ff30bfd-d709-4f3a-992a-d3b2f40135cc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.4943899018232819\n"
          ]
        }
      ],
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.05)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OgIYnQnCOQrh"
      },
      "source": [
        "#**Optimisation de la performance**"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Je vais opter pour trois approches afin de faire augmenter la performance:\n",
        "\n",
        "\n",
        "1.   Feature selection : Pearson Correlation\n",
        "2.   Reduction de la dimension : PCA\n",
        "3.   Natural Language Processing - NLP ( en prenant compte de la variable boilerplate )\n",
        "\n"
      ],
      "metadata": {
        "id": "VccFI7tkJr1u"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0v4O83KWYmHL"
      },
      "outputs": [],
      "source": [
        "df2 = sc.textFile(\"/train_noheader.tsv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aZcSqCAUhdUz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "ffa88c3b-e0bc-4cb8-c0d8-0e7852dab65b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"\\t\"4042\"\\t\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"\\t\"business\"\\t\"0.789131\"\\t\"2.055555556\"\\t\"0.676470588\"\\t\"0.205882353\"\\t\"0.047058824\"\\t\"0.023529412\"\\t\"0.443783175\"\\t\"0\"\\t\"0\"\\t\"0.09077381\"\\t\"0\"\\t\"0.245831182\"\\t\"0.003883495\"\\t\"1\"\\t\"1\"\\t\"24\"\\t\"0\"\\t\"5424\"\\t\"170\"\\t\"8\"\\t\"0.152941176\"\\t\"0.079129575\"\\t\"0\"'"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "df2.first()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7KK1l706iBG2",
        "outputId": "e634dcc5-c26b-4493-9822-61624fd9a63a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"', '\"4042\"', '\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"', '\"business\"', '\"0.789131\"', '\"2.055555556\"', '\"0.676470588\"', '\"0.205882353\"', '\"0.047058824\"', '\"0.023529412\"', '\"0.443783175\"', '\"0\"', '\"0\"', '\"0.09077381\"', '\"0\"', '\"0.245831182\"', '\"0.003883495\"', '\"1\"', '\"1\"', '\"24\"', '\"0\"', '\"5424\"', '\"170\"', '\"8\"', '\"0.152941176\"', '\"0.079129575\"', '\"0\"']\n"
          ]
        }
      ],
      "source": [
        "parts_it = df2.map(lambda row: row.split(\"\\t\"))\n",
        "print (parts_it.first())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U8uFqigcipJD",
        "outputId": "54732edf-eca4-45fd-d704-6b864c3eb967"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Row(url='\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"', urlid='\"4042\"', boilerplate='\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"', alchemy_category='\"business\"', alchemy_category_score='\"0.789131\"', avglinksize='\"2.055555556\"', commonlinkratio_1='\"0.676470588\"', commonlinkratio_2='\"0.205882353\"', commonlinkratio_3='\"0.047058824\"', commonlinkratio_4='\"0.023529412\"', compression_ratio='\"0.443783175\"', embed_ratio='\"0\"', framebased='\"0\"', frameTagRatio='\"0.09077381\"', hasDomainLink='\"0\"', html_ratio='\"0.245831182\"', image_ratio='\"0.003883495\"', is_news='\"1\"', lengthyLinkDomain='\"1\"', linkwordscore='\"24\"', news_front_page='\"0\"', non_markup_alphanum_characters='\"5424\"', numberOfLinks='\"170\"', numwords_in_url='\"8\"', parametrizedLinkRatio='\"0.152941176\"', spelling_errors_ratio='\"0.079129575\"', label='\"0\"')\n"
          ]
        }
      ],
      "source": [
        "data= parts_it.map(lambda p: Row (url=p[0], urlid=p[1], boilerplate=p[2],  alchemy_category=p[3], alchemy_category_score=(p[4]),\n",
        "                                 avglinksize=(p[5]), commonlinkratio_1=(p[6]), commonlinkratio_2=(p[7]), commonlinkratio_3=(p[8]), \n",
        "                                 commonlinkratio_4=(p[9]), compression_ratio=(p[10]), embed_ratio=(p[11]),\n",
        "                                 framebased=(p[12]), frameTagRatio=(p[13]), hasDomainLink=(p[14]), html_ratio=p[15], \n",
        "                                 image_ratio=(p[16]), is_news=(p[17]),lengthyLinkDomain=(p[18]), linkwordscore=(p[19]), \n",
        "                                 news_front_page=(p[20]),non_markup_alphanum_characters=(p[21]), numberOfLinks=(p[22]),\n",
        "                                 numwords_in_url=(p[23]), parametrizedLinkRatio=(p[24]),\n",
        "                                  spelling_errors_ratio=(p[25]), label=(p[26])))\n",
        "print(data.first())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "osfZyvdPlqLn"
      },
      "outputs": [],
      "source": [
        "data=spark.createDataFrame(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cGBGRsCdl1hs",
        "outputId": "7428611a-2086-4383-8ed7-4f207acd04a1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+------+--------------------+--------------------+----------------------+-------------+-----------------+-----------------+-----------------+-----------------+-----------------+-------------+----------+-------------+-------------+-------------+-------------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|                 url| urlid|         boilerplate|    alchemy_category|alchemy_category_score|  avglinksize|commonlinkratio_1|commonlinkratio_2|commonlinkratio_3|commonlinkratio_4|compression_ratio|  embed_ratio|framebased|frameTagRatio|hasDomainLink|   html_ratio|  image_ratio|is_news|lengthyLinkDomain|linkwordscore|news_front_page|non_markup_alphanum_characters|numberOfLinks|numwords_in_url|parametrizedLinkRatio|spelling_errors_ratio|label|\n",
            "+--------------------+------+--------------------+--------------------+----------------------+-------------+-----------------+-----------------+-----------------+-----------------+-----------------+-------------+----------+-------------+-------------+-------------+-------------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|\"http://www.bloom...|\"4042\"|\"{\"\"title\"\":\"\"IBM...|          \"business\"|            \"0.789131\"|\"2.055555556\"|    \"0.676470588\"|    \"0.205882353\"|    \"0.047058824\"|    \"0.023529412\"|    \"0.443783175\"|          \"0\"|       \"0\"| \"0.09077381\"|          \"0\"|\"0.245831182\"|\"0.003883495\"|    \"1\"|              \"1\"|         \"24\"|            \"0\"|                        \"5424\"|        \"170\"|            \"8\"|        \"0.152941176\"|        \"0.079129575\"|  \"0\"|\n",
            "|\"http://www.popsc...|\"8471\"|\"{\"\"title\"\":\"\"The...|        \"recreation\"|            \"0.574147\"|\"3.677966102\"|     \"0.50802139\"|    \"0.288770053\"|    \"0.213903743\"|    \"0.144385027\"|    \"0.468648998\"|          \"0\"|       \"0\"|\"0.098707403\"|          \"0\"|\"0.203489628\"|\"0.088652482\"|    \"1\"|              \"1\"|         \"40\"|            \"0\"|                        \"4973\"|        \"187\"|            \"9\"|        \"0.181818182\"|        \"0.125448029\"|  \"1\"|\n",
            "|\"http://www.mensh...|\"1164\"|\"{\"\"title\"\":\"\"Fru...|            \"health\"|            \"0.996526\"|\"2.382882883\"|    \"0.562015504\"|    \"0.321705426\"|    \"0.120155039\"|    \"0.042635659\"|    \"0.525448029\"|          \"0\"|       \"0\"|\"0.072447859\"|          \"0\"| \"0.22640177\"|\"0.120535714\"|    \"1\"|              \"1\"|         \"55\"|            \"0\"|                        \"2240\"|        \"258\"|           \"11\"|        \"0.166666667\"|        \"0.057613169\"|  \"1\"|\n",
            "|\"http://www.dumbl...|\"6684\"|\"{\"\"title\"\":\"\"10 ...|            \"health\"|            \"0.801248\"|\"1.543103448\"|            \"0.4\"|            \"0.1\"|    \"0.016666667\"|              \"0\"|    \"0.480724749\"|          \"0\"|       \"0\"|\"0.095860566\"|          \"0\"|\"0.265655744\"|\"0.035343035\"|    \"1\"|              \"0\"|         \"24\"|            \"0\"|                        \"2737\"|        \"120\"|            \"5\"|        \"0.041666667\"|        \"0.100858369\"|  \"1\"|\n",
            "|\"http://bleacherr...|\"9006\"|\"{\"\"title\"\":\"\"The...|            \"sports\"|            \"0.719157\"|\"2.676470588\"|            \"0.5\"|    \"0.222222222\"|     \"0.12345679\"|    \"0.043209877\"|    \"0.446143274\"|          \"0\"|       \"0\"|\"0.024908425\"|          \"0\"|\"0.228887247\"|\"0.050473186\"|    \"1\"|              \"1\"|         \"14\"|            \"0\"|                       \"12032\"|        \"162\"|           \"10\"|        \"0.098765432\"|        \"0.082568807\"|  \"0\"|\n",
            "|\"http://www.conve...|\"7018\"|\"{\"\"url\"\":\"\"conve...|                 \"?\"|                   \"?\"|        \"119\"|    \"0.745454545\"|    \"0.581818182\"|    \"0.290909091\"|    \"0.018181818\"|    \"0.434639175\"|          \"0\"|       \"0\"| \"0.01984127\"|          \"0\"|\"0.298299595\"|\"0.038636364\"|    \"?\"|              \"0\"|         \"12\"|            \"?\"|                        \"4368\"|         \"55\"|            \"3\"|        \"0.054545455\"|        \"0.087356322\"|  \"0\"|\n",
            "|\"http://gofashion...|\"8685\"|\"{\"\"title\"\":\"\"fas...|\"arts_entertainment\"|             \"0.22111\"|\"0.773809524\"|    \"0.215053763\"|    \"0.053763441\"|    \"0.043010753\"|    \"0.043010753\"|    \"0.579596413\"|          \"0\"|       \"0\"|\"0.039568345\"|          \"0\"|\"0.218978009\"|\"0.311377246\"|    \"1\"|              \"0\"|         \"21\"|            \"0\"|                        \"1287\"|         \"93\"|            \"3\"|        \"0.548387097\"|        \"0.064327485\"|  \"1\"|\n",
            "|\"http://www.insid...|\"3402\"|\"{\"\"url\"\":\"\"insid...|                 \"?\"|                   \"?\"|\"1.883333333\"|     \"0.71969697\"|    \"0.265151515\"|    \"0.113636364\"|    \"0.015151515\"|     \"0.49934811\"|          \"0\"|       \"0\"| \"0.02661597\"|          \"0\"|\"0.173745927\"|\"0.025830258\"|    \"?\"|              \"0\"|          \"5\"|            \"?\"|                       \"27656\"|        \"132\"|            \"4\"|        \"0.068181818\"|        \"0.148550725\"|  \"0\"|\n",
            "|\"http://www.valet...| \"477\"|\"{\"\"title\"\":\"\"Val...|                 \"?\"|                   \"?\"|\"0.471502591\"|    \"0.190721649\"|    \"0.036082474\"|              \"0\"|              \"0\"|    \"0.383199079\"|          \"0\"|       \"0\"|\"0.021705426\"|          \"0\"| \"0.11496229\"|\"1.136645963\"|    \"1\"|              \"0\"|         \"17\"|            \"0\"|                        \"2471\"|        \"194\"|            \"7\"|        \"0.644329897\"|              \"0.125\"|  \"1\"|\n",
            "|\"http://www.howsw...|\"6731\"|\"{\"\"url\"\":\"\"howsw...|                 \"?\"|                   \"?\"| \"2.41011236\"|    \"0.469325153\"|    \"0.101226994\"|    \"0.018404908\"|    \"0.003067485\"|    \"0.465859401\"|          \"0\"|       \"0\"|      \"0.012\"|          \"0\"|\"0.205116573\"| \"0.20626151\"|    \"?\"|              \"1\"|         \"14\"|            \"?\"|                       \"11459\"|        \"326\"|            \"4\"|        \"0.236196319\"|        \"0.094412331\"|  \"1\"|\n",
            "|\"http://www.reute...|\"2293\"|\"{\"\"title\"\":\"\"Bus...|          \"business\"|            \"0.816604\"|\"2.506527415\"|    \"0.637755102\"|    \"0.293367347\"|    \"0.091836735\"|    \"0.048469388\"|    \"0.592321755\"|          \"0\"|       \"0\"|\"0.056497175\"|          \"0\"|\"0.223003543\"|\"0.511363636\"|    \"1\"|              \"1\"|         \"53\"|            \"0\"|                        \"4401\"|        \"392\"|            \"0\"|        \"0.160714286\"|        \"0.073684211\"|  \"0\"|\n",
            "|\"http://www.midwe...|\"3651\"|\"{\"\"title\"\":\"\"A T...|            \"sports\"|             \"0.89156\"|\"4.986111111\"|           \"0.64\"|    \"0.426666667\"|           \"0.32\"|    \"0.293333333\"|    \"0.521064302\"|\"0.004065041\"|       \"0\"|\"0.162689805\"|          \"0\"| \"0.24691196\"| \"0.06097561\"|    \"?\"|              \"1\"|         \"40\"|            \"0\"|                        \"2701\"|         \"75\"|            \"8\"|        \"0.186666667\"|        \"0.115384615\"|  \"0\"|\n",
            "|\"http://www.ivill...|\"1917\"|\"{\"\"title\"\":\"\"9 F...|            \"health\"|            \"0.872323\"|\"3.056910569\"|    \"0.595588235\"|    \"0.227941176\"|    \"0.044117647\"|    \"0.014705882\"|    \"0.573109244\"|          \"0\"|       \"0\"|\"0.074576271\"|          \"0\"|\"0.236280941\"| \"0.08411215\"|    \"1\"|              \"1\"|         \"64\"|            \"0\"|                        \"1062\"|        \"136\"|            \"9\"|        \"0.169117647\"|        \"0.180327869\"|  \"1\"|\n",
            "|\"http://www.theda...|\"1063\"|\"{\"\"title\"\":\"\" \"\"...|                 \"?\"|                   \"?\"|          \"0\"|              \"0\"|              \"0\"|              \"0\"|              \"0\"|             \"21\"|         \"-1\"|       \"0\"|\"0.057692308\"|          \"0\"|\"0.328343661\"|         \"-1\"|    \"?\"|              \"0\"|          \"0\"|            \"0\"|                        \"4091\"|          \"5\"|           \"11\"|                  \"0\"|        \"0.083333333\"|  \"1\"|\n",
            "|\"http://www.phill...|\"2110\"|\"{\"\"url\"\":\"\"phill...|        \"recreation\"|            \"0.559991\"|\"2.299492386\"|    \"0.547413793\"|    \"0.206896552\"|    \"0.056034483\"|    \"0.017241379\"|    \"0.473964868\"|          \"0\"|       \"0\"|\"0.078431373\"|          \"1\"|\"0.205378622\"|\"0.036423841\"|    \"?\"|              \"1\"|         \"37\"|            \"0\"|                        \"3610\"|        \"232\"|           \"11\"|        \"0.215517241\"|        \"0.080204778\"|  \"0\"|\n",
            "|\"http://sportsill...|\"5318\"|\"{\"\"title\"\":\"\"Iza...|        \"recreation\"|            \"0.548963\"|\"0.990430622\"|    \"0.522522523\"|    \"0.108108108\"|    \"0.009009009\"|              \"0\"|    \"0.414154653\"|          \"0\"|       \"0\"|\"0.054992764\"|          \"0\"|\"0.189932886\"|\"0.075630252\"|    \"?\"|              \"0\"|         \"54\"|            \"0\"|                         \"883\"|        \"222\"|            \"4\"|        \"0.013513514\"|        \"0.059602649\"|  \"0\"|\n",
            "|\"http://theawesom...|\"9842\"|\"{\"\"title\"\":\"\"Liq...|        \"recreation\"|            \"0.598149\"|\"0.929824561\"|    \"0.068965517\"|              \"0\"|              \"0\"|              \"0\"|    \"0.478354978\"|          \"0\"|       \"0\"|\"0.157575758\"|          \"0\"|\"0.310008857\"|\"0.276315789\"|    \"1\"|              \"0\"|         \"50\"|            \"0\"|                         \"268\"|         \"58\"|            \"2\"|        \"0.137931034\"|        \"0.107142857\"|  \"0\"|\n",
            "|\"http://www.guard...|\"3946\"|\"{\"\"title\"\":null,...|\"arts_entertainment\"|             \"0.77292\"|\"2.083333333\"|    \"0.421052632\"|    \"0.178947368\"|    \"0.042105263\"|              \"0\"|    \"0.462994836\"|          \"0\"|       \"0\"|\"0.099778271\"|          \"0\"|\"0.232048045\"|\"0.275862069\"|    \"1\"|              \"0\"|         \"49\"|            \"0\"|                         \"852\"|         \"95\"|            \"8\"|        \"0.305263158\"|        \"0.139737991\"|  \"0\"|\n",
            "|\"http://sugarcraf...| \"334\"|\"{\"\"title\"\":\"\"Gri...|        \"recreation\"|            \"0.287884\"|\"3.164383562\"|    \"0.586206897\"|    \"0.275862069\"|    \"0.172413793\"|    \"0.149425287\"|    \"0.520414776\"|          \"0\"|       \"0\"|\"0.035864979\"|          \"0\"|\"0.265881624\"|\"0.091549296\"|    \"1\"|              \"1\"|         \"23\"|            \"0\"|                        \"3509\"|         \"87\"|            \"2\"|        \"0.402298851\"|         \"0.09057971\"|  \"1\"|\n",
            "|\"http://www.ivill...| \"271\"|\"{\"\"title\"\":\"\"How...|            \"health\"|            \"0.585389\"|\"2.826771654\"|    \"0.669064748\"|    \"0.237410072\"|    \"0.064748201\"|    \"0.021582734\"|     \"0.56279509\"|          \"0\"|       \"0\"| \"0.07345576\"|          \"0\"|\"0.247035291\"|\"0.053254438\"|    \"1\"|              \"1\"|         \"55\"|            \"0\"|                        \"1511\"|        \"139\"|            \"9\"|        \"0.165467626\"|        \"0.065217391\"|  \"1\"|\n",
            "+--------------------+------+--------------------+--------------------+----------------------+-------------+-----------------+-----------------+-----------------+-----------------+-----------------+-------------+----------+-------------+-------------+-------------+-------------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w4gqiRFCl_RD"
      },
      "source": [
        "### **Feature engineering**"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import regexp_replace\n",
        "from pyspark.sql.functions import *\n",
        "import numpy as np\n",
        "df=data\n",
        "for colName in data.columns :\n",
        "\n",
        "  df= df.withColumn(colName,regexp_replace(col(colName), '\"', \"\"))\n"
      ],
      "metadata": {
        "id": "AwUFkhmSXCBI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JZ_m0QF6cD3-",
        "outputId": "955494b3-e310-48bd-a3d0-c1a4f69a400c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|                 url|urlid|         boilerplate|  alchemy_category|alchemy_category_score|avglinksize|commonlinkratio_1|commonlinkratio_2|commonlinkratio_3|commonlinkratio_4|compression_ratio|embed_ratio|framebased|frameTagRatio|hasDomainLink| html_ratio|image_ratio|is_news|lengthyLinkDomain|linkwordscore|news_front_page|non_markup_alphanum_characters|numberOfLinks|numwords_in_url|parametrizedLinkRatio|spelling_errors_ratio|label|\n",
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|http://www.bloomb...| 4042|{title:IBM Sees H...|          business|              0.789131|2.055555556|      0.676470588|      0.205882353|      0.047058824|      0.023529412|      0.443783175|          0|         0|   0.09077381|            0|0.245831182|0.003883495|      1|                1|           24|              0|                          5424|          170|              8|          0.152941176|          0.079129575|    0|\n",
            "|http://www.popsci...| 8471|{title:The Fully ...|        recreation|              0.574147|3.677966102|       0.50802139|      0.288770053|      0.213903743|      0.144385027|      0.468648998|          0|         0|  0.098707403|            0|0.203489628|0.088652482|      1|                1|           40|              0|                          4973|          187|              9|          0.181818182|          0.125448029|    1|\n",
            "|http://www.menshe...| 1164|{title:Fruits tha...|            health|              0.996526|2.382882883|      0.562015504|      0.321705426|      0.120155039|      0.042635659|      0.525448029|          0|         0|  0.072447859|            0| 0.22640177|0.120535714|      1|                1|           55|              0|                          2240|          258|             11|          0.166666667|          0.057613169|    1|\n",
            "|http://www.dumbli...| 6684|{title:10 Foolpro...|            health|              0.801248|1.543103448|              0.4|              0.1|      0.016666667|                0|      0.480724749|          0|         0|  0.095860566|            0|0.265655744|0.035343035|      1|                0|           24|              0|                          2737|          120|              5|          0.041666667|          0.100858369|    1|\n",
            "|http://bleacherre...| 9006|{title:The 50 Coo...|            sports|              0.719157|2.676470588|              0.5|      0.222222222|       0.12345679|      0.043209877|      0.446143274|          0|         0|  0.024908425|            0|0.228887247|0.050473186|      1|                1|           14|              0|                         12032|          162|             10|          0.098765432|          0.082568807|    0|\n",
            "|http://www.conven...| 7018|{url:conveniencem...|                 ?|                     ?|        119|      0.745454545|      0.581818182|      0.290909091|      0.018181818|      0.434639175|          0|         0|   0.01984127|            0|0.298299595|0.038636364|      ?|                0|           12|              ?|                          4368|           55|              3|          0.054545455|          0.087356322|    0|\n",
            "|http://gofashionl...| 8685|{title:fashion la...|arts_entertainment|               0.22111|0.773809524|      0.215053763|      0.053763441|      0.043010753|      0.043010753|      0.579596413|          0|         0|  0.039568345|            0|0.218978009|0.311377246|      1|                0|           21|              0|                          1287|           93|              3|          0.548387097|          0.064327485|    1|\n",
            "|http://www.inside...| 3402|{url:insidersheal...|                 ?|                     ?|1.883333333|       0.71969697|      0.265151515|      0.113636364|      0.015151515|       0.49934811|          0|         0|   0.02661597|            0|0.173745927|0.025830258|      ?|                0|            5|              ?|                         27656|          132|              4|          0.068181818|          0.148550725|    0|\n",
            "|http://www.valetm...|  477|{title:Valet The ...|                 ?|                     ?|0.471502591|      0.190721649|      0.036082474|                0|                0|      0.383199079|          0|         0|  0.021705426|            0| 0.11496229|1.136645963|      1|                0|           17|              0|                          2471|          194|              7|          0.644329897|                0.125|    1|\n",
            "|http://www.howswe...| 6731|{url:howsweeteats...|                 ?|                     ?| 2.41011236|      0.469325153|      0.101226994|      0.018404908|      0.003067485|      0.465859401|          0|         0|        0.012|            0|0.205116573| 0.20626151|      ?|                1|           14|              ?|                         11459|          326|              4|          0.236196319|          0.094412331|    1|\n",
            "|http://www.reuter...| 2293|{title:Business F...|          business|              0.816604|2.506527415|      0.637755102|      0.293367347|      0.091836735|      0.048469388|      0.592321755|          0|         0|  0.056497175|            0|0.223003543|0.511363636|      1|                1|           53|              0|                          4401|          392|              0|          0.160714286|          0.073684211|    0|\n",
            "|http://www.midwes...| 3651|{title:A Tip of t...|            sports|               0.89156|4.986111111|             0.64|      0.426666667|             0.32|      0.293333333|      0.521064302|0.004065041|         0|  0.162689805|            0| 0.24691196| 0.06097561|      ?|                1|           40|              0|                          2701|           75|              8|          0.186666667|          0.115384615|    0|\n",
            "|http://www.ivilla...| 1917|{title:9 Foods Th...|            health|              0.872323|3.056910569|      0.595588235|      0.227941176|      0.044117647|      0.014705882|      0.573109244|          0|         0|  0.074576271|            0|0.236280941| 0.08411215|      1|                1|           64|              0|                          1062|          136|              9|          0.169117647|          0.180327869|    1|\n",
            "|http://www.thedai...| 1063|{title: ,body: ,u...|                 ?|                     ?|          0|                0|                0|                0|                0|               21|         -1|         0|  0.057692308|            0|0.328343661|         -1|      ?|                0|            0|              0|                          4091|            5|             11|                    0|          0.083333333|    1|\n",
            "|http://www.philly...| 2110|{url:phillyburbs ...|        recreation|              0.559991|2.299492386|      0.547413793|      0.206896552|      0.056034483|      0.017241379|      0.473964868|          0|         0|  0.078431373|            1|0.205378622|0.036423841|      ?|                1|           37|              0|                          3610|          232|             11|          0.215517241|          0.080204778|    0|\n",
            "|http://sportsillu...| 5318|{title:Izabel Gou...|        recreation|              0.548963|0.990430622|      0.522522523|      0.108108108|      0.009009009|                0|      0.414154653|          0|         0|  0.054992764|            0|0.189932886|0.075630252|      ?|                0|           54|              0|                           883|          222|              4|          0.013513514|          0.059602649|    0|\n",
            "|http://theawesome...| 9842|{title:Liquid Mou...|        recreation|              0.598149|0.929824561|      0.068965517|                0|                0|                0|      0.478354978|          0|         0|  0.157575758|            0|0.310008857|0.276315789|      1|                0|           50|              0|                           268|           58|              2|          0.137931034|          0.107142857|    0|\n",
            "|http://www.guardi...| 3946|{title:null,body:...|arts_entertainment|               0.77292|2.083333333|      0.421052632|      0.178947368|      0.042105263|                0|      0.462994836|          0|         0|  0.099778271|            0|0.232048045|0.275862069|      1|                0|           49|              0|                           852|           95|              8|          0.305263158|          0.139737991|    0|\n",
            "|http://sugarcraft...|  334|{title:Grilled Pe...|        recreation|              0.287884|3.164383562|      0.586206897|      0.275862069|      0.172413793|      0.149425287|      0.520414776|          0|         0|  0.035864979|            0|0.265881624|0.091549296|      1|                1|           23|              0|                          3509|           87|              2|          0.402298851|           0.09057971|    1|\n",
            "|http://www.ivilla...|  271|{title:How to Mak...|            health|              0.585389|2.826771654|      0.669064748|      0.237410072|      0.064748201|      0.021582734|       0.56279509|          0|         0|   0.07345576|            0|0.247035291|0.053254438|      1|                1|           55|              0|                          1511|          139|              9|          0.165467626|          0.065217391|    1|\n",
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col,when\n",
        "for colName in df.columns :\n",
        "      df=df.withColumn(colName, when(col(colName)==\"?\" ,None).otherwise(col(colName)))\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D6l3jUdxd6MZ",
        "outputId": "d324d8e3-a083-4aa0-c853-37636ebcf13a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|                 url|urlid|         boilerplate|  alchemy_category|alchemy_category_score|avglinksize|commonlinkratio_1|commonlinkratio_2|commonlinkratio_3|commonlinkratio_4|compression_ratio|embed_ratio|framebased|frameTagRatio|hasDomainLink| html_ratio|image_ratio|is_news|lengthyLinkDomain|linkwordscore|news_front_page|non_markup_alphanum_characters|numberOfLinks|numwords_in_url|parametrizedLinkRatio|spelling_errors_ratio|label|\n",
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "|http://www.bloomb...| 4042|{title:IBM Sees H...|          business|              0.789131|2.055555556|      0.676470588|      0.205882353|      0.047058824|      0.023529412|      0.443783175|          0|         0|   0.09077381|            0|0.245831182|0.003883495|      1|                1|           24|              0|                          5424|          170|              8|          0.152941176|          0.079129575|    0|\n",
            "|http://www.popsci...| 8471|{title:The Fully ...|        recreation|              0.574147|3.677966102|       0.50802139|      0.288770053|      0.213903743|      0.144385027|      0.468648998|          0|         0|  0.098707403|            0|0.203489628|0.088652482|      1|                1|           40|              0|                          4973|          187|              9|          0.181818182|          0.125448029|    1|\n",
            "|http://www.menshe...| 1164|{title:Fruits tha...|            health|              0.996526|2.382882883|      0.562015504|      0.321705426|      0.120155039|      0.042635659|      0.525448029|          0|         0|  0.072447859|            0| 0.22640177|0.120535714|      1|                1|           55|              0|                          2240|          258|             11|          0.166666667|          0.057613169|    1|\n",
            "|http://www.dumbli...| 6684|{title:10 Foolpro...|            health|              0.801248|1.543103448|              0.4|              0.1|      0.016666667|                0|      0.480724749|          0|         0|  0.095860566|            0|0.265655744|0.035343035|      1|                0|           24|              0|                          2737|          120|              5|          0.041666667|          0.100858369|    1|\n",
            "|http://bleacherre...| 9006|{title:The 50 Coo...|            sports|              0.719157|2.676470588|              0.5|      0.222222222|       0.12345679|      0.043209877|      0.446143274|          0|         0|  0.024908425|            0|0.228887247|0.050473186|      1|                1|           14|              0|                         12032|          162|             10|          0.098765432|          0.082568807|    0|\n",
            "|http://www.conven...| 7018|{url:conveniencem...|              null|                  null|        119|      0.745454545|      0.581818182|      0.290909091|      0.018181818|      0.434639175|          0|         0|   0.01984127|            0|0.298299595|0.038636364|   null|                0|           12|           null|                          4368|           55|              3|          0.054545455|          0.087356322|    0|\n",
            "|http://gofashionl...| 8685|{title:fashion la...|arts_entertainment|               0.22111|0.773809524|      0.215053763|      0.053763441|      0.043010753|      0.043010753|      0.579596413|          0|         0|  0.039568345|            0|0.218978009|0.311377246|      1|                0|           21|              0|                          1287|           93|              3|          0.548387097|          0.064327485|    1|\n",
            "|http://www.inside...| 3402|{url:insidersheal...|              null|                  null|1.883333333|       0.71969697|      0.265151515|      0.113636364|      0.015151515|       0.49934811|          0|         0|   0.02661597|            0|0.173745927|0.025830258|   null|                0|            5|           null|                         27656|          132|              4|          0.068181818|          0.148550725|    0|\n",
            "|http://www.valetm...|  477|{title:Valet The ...|              null|                  null|0.471502591|      0.190721649|      0.036082474|                0|                0|      0.383199079|          0|         0|  0.021705426|            0| 0.11496229|1.136645963|      1|                0|           17|              0|                          2471|          194|              7|          0.644329897|                0.125|    1|\n",
            "|http://www.howswe...| 6731|{url:howsweeteats...|              null|                  null| 2.41011236|      0.469325153|      0.101226994|      0.018404908|      0.003067485|      0.465859401|          0|         0|        0.012|            0|0.205116573| 0.20626151|   null|                1|           14|           null|                         11459|          326|              4|          0.236196319|          0.094412331|    1|\n",
            "|http://www.reuter...| 2293|{title:Business F...|          business|              0.816604|2.506527415|      0.637755102|      0.293367347|      0.091836735|      0.048469388|      0.592321755|          0|         0|  0.056497175|            0|0.223003543|0.511363636|      1|                1|           53|              0|                          4401|          392|              0|          0.160714286|          0.073684211|    0|\n",
            "|http://www.midwes...| 3651|{title:A Tip of t...|            sports|               0.89156|4.986111111|             0.64|      0.426666667|             0.32|      0.293333333|      0.521064302|0.004065041|         0|  0.162689805|            0| 0.24691196| 0.06097561|   null|                1|           40|              0|                          2701|           75|              8|          0.186666667|          0.115384615|    0|\n",
            "|http://www.ivilla...| 1917|{title:9 Foods Th...|            health|              0.872323|3.056910569|      0.595588235|      0.227941176|      0.044117647|      0.014705882|      0.573109244|          0|         0|  0.074576271|            0|0.236280941| 0.08411215|      1|                1|           64|              0|                          1062|          136|              9|          0.169117647|          0.180327869|    1|\n",
            "|http://www.thedai...| 1063|{title: ,body: ,u...|              null|                  null|          0|                0|                0|                0|                0|               21|         -1|         0|  0.057692308|            0|0.328343661|         -1|   null|                0|            0|              0|                          4091|            5|             11|                    0|          0.083333333|    1|\n",
            "|http://www.philly...| 2110|{url:phillyburbs ...|        recreation|              0.559991|2.299492386|      0.547413793|      0.206896552|      0.056034483|      0.017241379|      0.473964868|          0|         0|  0.078431373|            1|0.205378622|0.036423841|   null|                1|           37|              0|                          3610|          232|             11|          0.215517241|          0.080204778|    0|\n",
            "|http://sportsillu...| 5318|{title:Izabel Gou...|        recreation|              0.548963|0.990430622|      0.522522523|      0.108108108|      0.009009009|                0|      0.414154653|          0|         0|  0.054992764|            0|0.189932886|0.075630252|   null|                0|           54|              0|                           883|          222|              4|          0.013513514|          0.059602649|    0|\n",
            "|http://theawesome...| 9842|{title:Liquid Mou...|        recreation|              0.598149|0.929824561|      0.068965517|                0|                0|                0|      0.478354978|          0|         0|  0.157575758|            0|0.310008857|0.276315789|      1|                0|           50|              0|                           268|           58|              2|          0.137931034|          0.107142857|    0|\n",
            "|http://www.guardi...| 3946|{title:null,body:...|arts_entertainment|               0.77292|2.083333333|      0.421052632|      0.178947368|      0.042105263|                0|      0.462994836|          0|         0|  0.099778271|            0|0.232048045|0.275862069|      1|                0|           49|              0|                           852|           95|              8|          0.305263158|          0.139737991|    0|\n",
            "|http://sugarcraft...|  334|{title:Grilled Pe...|        recreation|              0.287884|3.164383562|      0.586206897|      0.275862069|      0.172413793|      0.149425287|      0.520414776|          0|         0|  0.035864979|            0|0.265881624|0.091549296|      1|                1|           23|              0|                          3509|           87|              2|          0.402298851|           0.09057971|    1|\n",
            "|http://www.ivilla...|  271|{title:How to Mak...|            health|              0.585389|2.826771654|      0.669064748|      0.237410072|      0.064748201|      0.021582734|       0.56279509|          0|         0|   0.07345576|            0|0.247035291|0.053254438|      1|                1|           55|              0|                          1511|          139|              9|          0.165467626|          0.065217391|    1|\n",
            "+--------------------+-----+--------------------+------------------+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j5o18uHel7un",
        "outputId": "9f95d349-b670-4a3c-bf4e-ca9f9da7d8c3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----------------+------------------+\n",
            "|         Column_Name|Null_Values_Count|Null_Value_Percent|\n",
            "+--------------------+-----------------+------------------+\n",
            "|    alchemy_category|             2342|31.670047329276542|\n",
            "|alchemy_category_...|             2342|31.670047329276542|\n",
            "|             is_news|             2843|  38.4448951994591|\n",
            "|     news_front_page|             1248|16.876267748478703|\n",
            "+--------------------+-----------------+------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.sql.functions import *\n",
        "\n",
        "def null_value_calc(df):\n",
        "    null_columns_counts = []\n",
        "    numRows = df.count()\n",
        "    for k in df.columns:\n",
        "        nullRows = df.where(col(k).isNull()).count()\n",
        "        temp = k,nullRows,(nullRows/numRows)*100\n",
        "        if nullRows>0:\n",
        "               null_columns_counts.append(temp)\n",
        "    return(null_columns_counts)\n",
        "\n",
        "null_columns_calc_list = null_value_calc(df)\n",
        "spark.createDataFrame(null_columns_calc_list, ['Column_Name', 'Null_Values_Count','Null_Value_Percent']).show()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.select('alchemy_category_score', 'alchemy_category', 'is_news', 'news_front_page').summary('mean', '50%', 'max').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LAnzykkFfe5p",
        "outputId": "87eb8b92-4967-4ed0-c3f5-4e0e147dd2b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+----------------------+----------------+-------+-------------------+\n",
            "|summary|alchemy_category_score|alchemy_category|is_news|    news_front_page|\n",
            "+-------+----------------------+----------------+-------+-------------------+\n",
            "|   mean|    0.6033343166237878|            null|    1.0|0.04782820888238165|\n",
            "|    50%|              0.625616|            null|    1.0|                0.0|\n",
            "|    max|              0.999426|         weather|      1|                  1|\n",
            "+-------+----------------------+----------------+-------+-------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.fillna({'alchemy_category_score':0.6033343166237878, 'alchemy_category':'weather', 'is_news':0, 'news_front_page':0.04782820888238165})"
      ],
      "metadata": {
        "id": "NxXwppOrirF6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop('url', 'urlid', 'boilerplate')"
      ],
      "metadata": {
        "id": "EF_u8pXVjdD-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.feature import StringIndexer\n",
        "ac_indexer = StringIndexer(inputCol=\"alchemy_category\", outputCol=\"alchemi_category\")\n",
        "#Fits a model to the input dataset with optional parameters.\n",
        "data = ac_indexer.fit(df).transform(df)\n",
        "data=data.drop('alchemy_category')\n",
        "data.select('alchemi_category').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qk7ZT8E7kN5V",
        "outputId": "e79ffb11-c936-4100-dd55-6ce42e0b5f1d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------+\n",
            "|alchemi_category|\n",
            "+----------------+\n",
            "|             3.0|\n",
            "|             1.0|\n",
            "|             4.0|\n",
            "|             4.0|\n",
            "|             5.0|\n",
            "|             0.0|\n",
            "|             2.0|\n",
            "|             0.0|\n",
            "|             0.0|\n",
            "|             0.0|\n",
            "|             3.0|\n",
            "|             5.0|\n",
            "|             4.0|\n",
            "|             0.0|\n",
            "|             1.0|\n",
            "|             1.0|\n",
            "|             1.0|\n",
            "|             2.0|\n",
            "|             1.0|\n",
            "|             4.0|\n",
            "+----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.show(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oWdSwCvX2LAa",
        "outputId": "1bb8c90b-91f4-4fb0-d64c-aee197e50b47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "|alchemy_category_score|avglinksize|commonlinkratio_1|commonlinkratio_2|commonlinkratio_3|commonlinkratio_4|compression_ratio|embed_ratio|framebased|frameTagRatio|hasDomainLink| html_ratio|image_ratio|is_news|lengthyLinkDomain|linkwordscore|news_front_page|non_markup_alphanum_characters|numberOfLinks|numwords_in_url|parametrizedLinkRatio|spelling_errors_ratio|label|alchemi_category|\n",
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "|              0.789131|2.055555556|      0.676470588|      0.205882353|      0.047058824|      0.023529412|      0.443783175|          0|         0|   0.09077381|            0|0.245831182|0.003883495|      1|                1|           24|              0|                          5424|          170|              8|          0.152941176|          0.079129575|    0|             3.0|\n",
            "|              0.574147|3.677966102|       0.50802139|      0.288770053|      0.213903743|      0.144385027|      0.468648998|          0|         0|  0.098707403|            0|0.203489628|0.088652482|      1|                1|           40|              0|                          4973|          187|              9|          0.181818182|          0.125448029|    1|             1.0|\n",
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data=data.drop('framebased')"
      ],
      "metadata": {
        "id": "kblmIDJK2RXS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Approche 1: Feature selection 🔔**"
      ],
      "metadata": {
        "id": "yc8a_lWQmdLk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def convert(row):\n",
        "    Liste = []\n",
        "    for x in row:\n",
        "        Liste += [float(x)]\n",
        "    return Liste"
      ],
      "metadata": {
        "id": "6MggmgqnQGRH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.linalg import Vector as MLVector, Vectors as MLVectors\n",
        "from pyspark.mllib.linalg import Vector as MLLibVector, Vectors as MLLibVectors\n",
        "from pyspark.mllib.regression import  LabeledPoint\n",
        "\n",
        "labeled_data = data.rdd.map(lambda row : LabeledPoint(int(row[-2]),convert(row)[:-2],))"
      ],
      "metadata": {
        "id": "Sv4s-GTumb4j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labeled_data.first()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tsmnpleyk5oW",
        "outputId": "e16c3e8e-5280-4927-e1fb-8c08dd8151ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabeledPoint(0.0, [0.789131,2.055555556,0.676470588,0.205882353,0.047058824,0.023529412,0.443783175,0.0,0.09077381,0.0,0.245831182,0.003883495,1.0,1.0,24.0,0.0,5424.0,170.0,8.0,0.152941176,0.079129575])"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **Pearson correlation coefficients**"
      ],
      "metadata": {
        "id": "8u5jYYypsVm3"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating first Pearson correlation coefficients (PCCs) between the class and each features (scoreClass), then the PCCs between every pair of feature (scoreMatrix). Once these intermediate results are completed, I proceed into performing the feature selection. "
      ],
      "metadata": {
        "id": "w4zKZPZ-sidY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here below I define the functions used during the map and reduce stage of the distributed calculations."
      ],
      "metadata": {
        "id": "Zh5KGE-nt3_3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.stats.stats import pearsonr\n",
        "from sklearn.metrics import normalized_mutual_info_score\n",
        "from sklearn.feature_selection import mutual_info_regression\n",
        "import numpy as np\n",
        "\n",
        "#This function creates a list of k,v tuples, one per each label-feature combination. \n",
        "#'k' corresponds to the index of the feature and 'v' corresponds to a tuple of two elements: value of the label, value of the feature\n",
        "#lp : labeledPoint\n",
        "def meltLPclass(lp):\n",
        "    \n",
        "    label = lp.label\n",
        "    features = lp.features\n",
        "    r = range(features.size)\n",
        "    return [(i, (label, features[i])) for i in r]\n"
      ],
      "metadata": {
        "id": "PyH7UkZZshzU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This function calculates the Pearson correlation coefficient among two variables. It returns the index of the feature and its correlation coefficient. the parameter x is tuple "
      ],
      "metadata": {
        "id": "UqCaPAYKKs3Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def corr(x):\n",
        "    idx = x[0]\n",
        "    values = list(x[1])\n",
        "    \n",
        "    l = list(values)\n",
        "    v1, v2 = zip(*values)\n",
        "    p = pearsonr(v1, v2)[0]\n",
        "    \n",
        "    return (idx, p)"
      ],
      "metadata": {
        "id": "RD9FcI2NuCFi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This function calculates the Mutual information between two discrete variables. It relies on the 'adjusted_mutual_info_score' function available in the sklearn package."
      ],
      "metadata": {
        "id": "YvZiRP60K6U3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def mi(x):\n",
        "    idx = x[0]\n",
        "    values = list(x[1])\n",
        "    \n",
        "    l = list(values)\n",
        "    v1, v2 = zip(*values)\n",
        "    res = normalized_mutual_info_score(v1, v2)\n",
        "    \n",
        "    return (idx, res)"
      ],
      "metadata": {
        "id": "Qqxhs9HWsU6Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This function calculates the Mutual information between two continuous variables. It relies on the 'mutual_info_regression' function available in the sklearn package."
      ],
      "metadata": {
        "id": "kwb9sdcTLCLG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def miCont(x):\n",
        "    idx = x[0]\n",
        "    values = list(x[1])\n",
        "    \n",
        "    l = list(values)\n",
        "    v1, v2 = zip(*values)\n",
        "    V1 = np.array(v1).reshape(len(v1),1)  # 1-column matrix layout transformation\n",
        "    res = mutual_info_regression(V1, v2, discrete_features=False, random_state=42)[0]  # random_state is set to provide deterministic results\n",
        "    \n",
        "    return (idx, res)"
      ],
      "metadata": {
        "id": "vGGtCr2dsT2V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fscores = labeled_data.flatMap(meltLPclass).groupByKey().map(corr).collect()"
      ],
      "metadata": {
        "id": "d4OFeXvfsOJW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fscores[0:4]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VRv4YeKIud70",
        "outputId": "1124c268-59ed-4a7f-a2f2-943521a22356"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0, -0.051758018388009884),\n",
              " (2, 0.0833636926283919),\n",
              " (4, 0.10596419450867221),\n",
              " (6, -0.05973654614063667)]"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ncols = labeled_data.first().features.size  # number of columns (no class) of the dataset\n",
        "fscoresIdx, fscoresScore = zip(*fscores)\n",
        "scoreClass = [fscoresScore[fscoresIdx.index(i)] for i in range(ncols)]"
      ],
      "metadata": {
        "id": "d5oWfGjounz1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fscoresIdx[0:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W3DsjlMeu2Ng",
        "outputId": "28d3a2ff-8701-47f8-d891-ada33bb7f698"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0, 2, 4)"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fscoresScore[0:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3F5YwsSHu7Je",
        "outputId": "7223c54e-df8a-4767-8f47-df44e1201f11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(-0.051758018388009884, 0.0833636926283919, 0.10596419450867221)"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scoreClass[0:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vr4-6d_Iu9sz",
        "outputId": "ef40cd35-3cb2-4309-9efe-4f4136031dff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[-0.051758018388009884, 0.006172207286742423, 0.0833636926283919]"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### **Top n-features**"
      ],
      "metadata": {
        "id": "rgRf_chDvHqS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "nfs = 10 # number of feature to select"
      ],
      "metadata": {
        "id": "hKePvgtGvAf3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "L=[]\n",
        "for i in range (len(scoreClass)):\n",
        "  if scoreClass[i]< 0:\n",
        "    tup=(scoreClass[i]*(-1),i)\n",
        "  else:\n",
        "    tup=(scoreClass[i],i)\n",
        "  if scoreClass[i]!='nan':\n",
        "     L.append(tup)\n",
        "l=sorted(L,key=lambda tup: tup[0], reverse=True)\n",
        "fsIdx = [x[1] for x in l[0:nfs]]"
      ],
      "metadata": {
        "id": "PGqXytgJwdeF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fsIdx[0:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hPT5LMz5xp0E",
        "outputId": "053397fe-9fbc-41b1-89d5-5ede060467da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[8, 14, 4, 16, 3, 2, 5, 17, 6, 20]"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.linalg import Vectors\n",
        "\n",
        "def reduceLP(lp, fsIdx):\n",
        "    label = lp.label\n",
        "    features = lp.features\n",
        "    v = [features[i] for i in fsIdx]\n",
        "    return LabeledPoint(label, Vectors.dense(v))\n",
        "\n",
        "rddFS = labeled_data.map(lambda x: reduceLP(x, fsIdx))"
      ],
      "metadata": {
        "id": "uXgBXoYkzrpA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rddFS.first()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "56tRCwnezvvj",
        "outputId": "e94f57e1-007c-486a-ca47-f6fa17556ad2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabeledPoint(0.0, [0.09077381,24.0,0.047058824,5424.0,0.205882353,0.676470588,0.023529412,170.0,0.443783175,0.079129575])"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oJZgvoWjlDnu"
      },
      "outputs": [],
      "source": [
        "(training , test) =rddFS.randomSplit([0.8,0.2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bhJBGq1elZrP",
        "outputId": "fbcb71c3-990c-40f4-9299-c4156743d237"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.49511173184357543\n"
          ]
        }
      ],
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.005)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.05)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lG_4EWhG3_W1",
        "outputId": "d9c60751-cb8f-45b6-96c8-bfa0f82ec3a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.520682862770847\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Conclusion 1** : Feature selection using pearson correlation coefficients didn't really give us an improvement on the accuracy of both models ! So let's give a try to the second approch"
      ],
      "metadata": {
        "id": "P8weDNEsLf9N"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Approche 2: PCA 🔔**"
      ],
      "metadata": {
        "id": "-0nfzUjQMhlc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data.show(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xp-q4bZZNBs5",
        "outputId": "1fee8884-8f7b-4d47-dd75-c6775fd5bfbf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "|alchemy_category_score|avglinksize|commonlinkratio_1|commonlinkratio_2|commonlinkratio_3|commonlinkratio_4|compression_ratio|embed_ratio|frameTagRatio|hasDomainLink| html_ratio|image_ratio|is_news|lengthyLinkDomain|linkwordscore|news_front_page|non_markup_alphanum_characters|numberOfLinks|numwords_in_url|parametrizedLinkRatio|spelling_errors_ratio|label|alchemi_category|\n",
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "|              0.789131|2.055555556|      0.676470588|      0.205882353|      0.047058824|      0.023529412|      0.443783175|          0|   0.09077381|            0|0.245831182|0.003883495|      1|                1|           24|              0|                          5424|          170|              8|          0.152941176|          0.079129575|    0|             3.0|\n",
            "|              0.574147|3.677966102|       0.50802139|      0.288770053|      0.213903743|      0.144385027|      0.468648998|          0|  0.098707403|            0|0.203489628|0.088652482|      1|                1|           40|              0|                          4973|          187|              9|          0.181818182|          0.125448029|    1|             1.0|\n",
            "+----------------------+-----------+-----------------+-----------------+-----------------+-----------------+-----------------+-----------+-------------+-------------+-----------+-----------+-------+-----------------+-------------+---------------+------------------------------+-------------+---------------+---------------------+---------------------+-----+----------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import col\n",
        "from pyspark.sql.types import FloatType\n",
        "for i in data.columns:\n",
        "  data=data.withColumn(i, col(i).cast(FloatType()))"
      ],
      "metadata": {
        "id": "BKBHwktJRV6t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qUuudM16Rs7Z",
        "outputId": "1cbe9ba8-661d-410d-92af-b32170a56efe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- alchemy_category_score: float (nullable = true)\n",
            " |-- avglinksize: float (nullable = true)\n",
            " |-- commonlinkratio_1: float (nullable = true)\n",
            " |-- commonlinkratio_2: float (nullable = true)\n",
            " |-- commonlinkratio_3: float (nullable = true)\n",
            " |-- commonlinkratio_4: float (nullable = true)\n",
            " |-- compression_ratio: float (nullable = true)\n",
            " |-- embed_ratio: float (nullable = true)\n",
            " |-- frameTagRatio: float (nullable = true)\n",
            " |-- hasDomainLink: float (nullable = true)\n",
            " |-- html_ratio: float (nullable = true)\n",
            " |-- image_ratio: float (nullable = true)\n",
            " |-- is_news: float (nullable = true)\n",
            " |-- lengthyLinkDomain: float (nullable = true)\n",
            " |-- linkwordscore: float (nullable = true)\n",
            " |-- news_front_page: float (nullable = true)\n",
            " |-- non_markup_alphanum_characters: float (nullable = true)\n",
            " |-- numberOfLinks: float (nullable = true)\n",
            " |-- numwords_in_url: float (nullable = true)\n",
            " |-- parametrizedLinkRatio: float (nullable = true)\n",
            " |-- spelling_errors_ratio: float (nullable = true)\n",
            " |-- label: float (nullable = true)\n",
            " |-- alchemi_category: float (nullable = false)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(training , test) =data.randomSplit([0.8,0.2])"
      ],
      "metadata": {
        "id": "chvxMtaLNTRK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Cast to vectors so that PCA can work with our data\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "ignore = ['label']\n",
        "\n",
        "assembler = VectorAssembler(\n",
        "    inputCols=[x for x in training.columns if x not in ignore],\n",
        "    outputCol='features')\n",
        "\n",
        "train_vec = assembler.transform(training).select(['label', 'features'])\n",
        "test_vec = assembler.transform(test).select(['label', 'features'])"
      ],
      "metadata": {
        "id": "LLVqWK3zO7lM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.feature import MinMaxScaler\n",
        "scaler = MinMaxScaler(inputCol=\"features\", outputCol=\"scaledFeatures\")\n",
        "\n",
        "# Compute summary statistics and generate MinMaxScalerModel\n",
        "scalerModel = scaler.fit(train_vec)\n",
        "\n",
        "# rescale each feature to range [min, max].\n",
        "Trainscaled_data = scalerModel.transform(train_vec)\n",
        "# Compute summary statistics and generate MinMaxScalerModel\n",
        "#scalerModel = scaler.fit(test_vec)\n",
        "\n",
        "# rescale each feature to range [min, max].\n",
        "testscaled_data = scalerModel.transform(test_vec)"
      ],
      "metadata": {
        "id": "NLtNDFvrxCZf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Perform PCA\n",
        "from pyspark.ml.feature import PCA\n",
        "\n",
        "pca = PCA(k=10, inputCol=\"scaledFeatures\", outputCol = \"pca_features\")\n",
        "\n",
        "mdl_pca = pca.fit(Trainscaled_data)\n",
        "train_pca = mdl_pca.transform(Trainscaled_data)\n",
        "test_pca = mdl_pca.transform(testscaled_data)\n",
        "\n",
        "mdl_pca.explainedVariance"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ebZ8iFFKMkh9",
        "outputId": "b2c0941d-3285-4829-b14f-3ffc4736f8b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DenseVector([0.3015, 0.2209, 0.1304, 0.0576, 0.0521, 0.0459, 0.0395, 0.0359, 0.0328, 0.0225])"
            ]
          },
          "metadata": {},
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_pca.show(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f6HmA9q_0d_U",
        "outputId": "63c2b926-9870-4e77-8f19-ee88e3512d59"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+--------------------+--------------------+\n",
            "|label|            features|      scaledFeatures|        pca_features|\n",
            "+-----+--------------------+--------------------+--------------------+\n",
            "|  1.0|[0.07840909808874...|[0.00815836159370...|[-0.0650823735728...|\n",
            "|  1.0|[0.07840909808874...|[0.00815836159370...|[-0.8158975305454...|\n",
            "+-----+--------------------+--------------------+--------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_pca=train_pca.drop('features')\n",
        "test_pca=test_pca.drop('features')\n",
        "train_pca=train_pca.drop('scaledFeatures')\n",
        "test_pca=test_pca.drop('scaledFeatures')"
      ],
      "metadata": {
        "id": "HJsMK_SSTzZj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_pca.show(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aa39AaI4Ud07",
        "outputId": "0ee96a66-21d6-4f8e-97b4-b1c5f6df8a92"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+\n",
            "|label|        pca_features|\n",
            "+-----+--------------------+\n",
            "|  1.0|[-0.0650823735728...|\n",
            "|  1.0|[-0.8158975305454...|\n",
            "+-----+--------------------+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training = train_pca.rdd.map(lambda row : LabeledPoint(int(row[0]),MLLibVectors.fromML(row[1]),))"
      ],
      "metadata": {
        "id": "H2lB_AB7TRXU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test=test_pca.rdd.map(lambda row : LabeledPoint(int(row[0]),MLLibVectors.fromML(row[1]),))"
      ],
      "metadata": {
        "id": "_5GWQ74IUQmX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training.first()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qLZH7JRRUWp_",
        "outputId": "312bc833-0034-4f72-f2f2-05402b47c788"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabeledPoint(1.0, [-0.06508237357285881,0.09013824837629196,-0.1579084128297016,0.35099565683783807,0.18773342000336046,-0.10465300211225166,-0.0001622720656552501,-0.10464364920685171,0.008907928337961432,-0.010285004811913655])"
            ]
          },
          "metadata": {},
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.05, regType='l2', miniBatchFraction=0.01)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mUdzFThWTB1n",
        "outputId": "bef38e6c-5b39-4bfb-ec6e-d75b6bdb8a63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.6205766710353866\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.05, regType='l2', miniBatchFraction=0.01)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cBLFGiP9WEN8",
        "outputId": "ecf1709a-d55a-43ec-c43a-abcb80ed0c2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.5563564875491481\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##### **What could be the best value of regParam that could give us the maximum accuracy in this case ?**"
      ],
      "metadata": {
        "id": "9CL5jQ7-WR5M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def modelWithSVM(trainingData, validationData):\n",
        "  reg=[ 0.001, 0.005,0.01, 0.002, 0.0009]\n",
        "  bestSVMModel = None\n",
        "  bestAccuracy = 0\n",
        "  numOfIterations = 1000\n",
        "  visualizationData = []\n",
        "  for regularizer in reg:\n",
        "    model = SVMWithSGD.train(trainingData, numOfIterations, regParam=regularizer, miniBatchFraction=0.01)\n",
        "    predict = validationData.map(lambda ad: (ad.label, model.predict(ad.features)))\n",
        "    totalValidationAds = validationData.count()\n",
        "    correctlyPredicted = predict.filter(lambda x: x[0] == x[1]).count()\n",
        "    accuracy = float(correctlyPredicted)/totalValidationAds\n",
        "    visualizationData += [(regularizer, accuracy)]\n",
        "    if accuracy > bestAccuracy:\n",
        "      bestAccuracy = accuracy\n",
        "      bestSVMModel = model\n",
        "  return(visualizationData)"
      ],
      "metadata": {
        "id": "ZMpHoAeKWMdD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(modelWithSVM(training, test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y12mTAtujaVv",
        "outputId": "01525a83-1a45-4b9f-ed0c-3cbd658a862e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[(0.001, 0.6035386631716907), (0.005, 0.6055045871559633), (0.01, 0.59043250327654), (0.002, 0.6120576671035387), (0.0009, 0.6035386631716907)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "on peut remarquer que regparam=0.002 donne la meilleure performance. La performance a augmenté de 55% à 61%"
      ],
      "metadata": {
        "id": "V34Ua43bIKBd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.002, regType='l2', miniBatchFraction=0.01)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZhBR5FLJITz9",
        "outputId": "957abce4-2fff-4231-d157-8fa9b4da2ca4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.6120576671035387\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.002, regType='l2', miniBatchFraction=0.01)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmrRVbJxI1Ey",
        "outputId": "4184e6c8-40c4-49ce-af4b-7907a9b556ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.6297509829619922\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Conclusion 2**: PCA has improved the accuracy of both models : \n",
        "-> 49% to 62% 🙂"
      ],
      "metadata": {
        "id": "rcWLIaV6MZhe"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Approche 3 : NLP 🔔**"
      ],
      "metadata": {
        "id": "5Brb3EgLJgBh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9WRk8jFJOYUD"
      },
      "outputs": [],
      "source": [
        "df2 = sc.textFile(\"/train_noheader.tsv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "37bc1cd1-c898-44c4-b842-26b9e8331259",
        "id": "lrhzzRZxOYUI"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"\\t\"4042\"\\t\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"\\t\"business\"\\t\"0.789131\"\\t\"2.055555556\"\\t\"0.676470588\"\\t\"0.205882353\"\\t\"0.047058824\"\\t\"0.023529412\"\\t\"0.443783175\"\\t\"0\"\\t\"0\"\\t\"0.09077381\"\\t\"0\"\\t\"0.245831182\"\\t\"0.003883495\"\\t\"1\"\\t\"1\"\\t\"24\"\\t\"0\"\\t\"5424\"\\t\"170\"\\t\"8\"\\t\"0.152941176\"\\t\"0.079129575\"\\t\"0\"'"
            ]
          },
          "metadata": {},
          "execution_count": 166
        }
      ],
      "source": [
        "df2.first()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f595085-9a01-46c9-eef3-1e8addd89f06",
        "id": "oG4jZmkKOYUJ"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"', '\"4042\"', '\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"', '\"business\"', '\"0.789131\"', '\"2.055555556\"', '\"0.676470588\"', '\"0.205882353\"', '\"0.047058824\"', '\"0.023529412\"', '\"0.443783175\"', '\"0\"', '\"0\"', '\"0.09077381\"', '\"0\"', '\"0.245831182\"', '\"0.003883495\"', '\"1\"', '\"1\"', '\"24\"', '\"0\"', '\"5424\"', '\"170\"', '\"8\"', '\"0.152941176\"', '\"0.079129575\"', '\"0\"']\n"
          ]
        }
      ],
      "source": [
        "parts_it = df2.map(lambda row: row.split(\"\\t\"))\n",
        "print (parts_it.first())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eef001d5-d21a-406a-bf44-2c725193d952",
        "id": "EUZaFx7_OYUJ"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Row(url='\"http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html\"', urlid='\"4042\"', boilerplate='\"{\"\"title\"\":\"\"IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries\"\",\"\"body\"\":\"\"A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\\\u2019s what international business machines corp. sees in its crystal ball.\"\",\"\"url\"\":\"\"bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html\"\"}\"', alchemy_category='\"business\"')\n"
          ]
        }
      ],
      "source": [
        "data= parts_it.map(lambda p: Row (url=p[0], urlid=p[1], boilerplate=p[2],  alchemy_category=p[3]))\n",
        "print(data.first())\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HXfhjDUzOYUj"
      },
      "outputs": [],
      "source": [
        "data_nlp=spark.createDataFrame(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b594a3a0-f2b2-421f-8838-30d89f7ca2ec",
        "id": "krXULHWFOYUj"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+------+--------------------+--------------------+\n",
            "|                 url| urlid|         boilerplate|    alchemy_category|\n",
            "+--------------------+------+--------------------+--------------------+\n",
            "|\"http://www.bloom...|\"4042\"|\"{\"\"title\"\":\"\"IBM...|          \"business\"|\n",
            "|\"http://www.popsc...|\"8471\"|\"{\"\"title\"\":\"\"The...|        \"recreation\"|\n",
            "|\"http://www.mensh...|\"1164\"|\"{\"\"title\"\":\"\"Fru...|            \"health\"|\n",
            "|\"http://www.dumbl...|\"6684\"|\"{\"\"title\"\":\"\"10 ...|            \"health\"|\n",
            "|\"http://bleacherr...|\"9006\"|\"{\"\"title\"\":\"\"The...|            \"sports\"|\n",
            "|\"http://www.conve...|\"7018\"|\"{\"\"url\"\":\"\"conve...|                 \"?\"|\n",
            "|\"http://gofashion...|\"8685\"|\"{\"\"title\"\":\"\"fas...|\"arts_entertainment\"|\n",
            "|\"http://www.insid...|\"3402\"|\"{\"\"url\"\":\"\"insid...|                 \"?\"|\n",
            "|\"http://www.valet...| \"477\"|\"{\"\"title\"\":\"\"Val...|                 \"?\"|\n",
            "|\"http://www.howsw...|\"6731\"|\"{\"\"url\"\":\"\"howsw...|                 \"?\"|\n",
            "|\"http://www.reute...|\"2293\"|\"{\"\"title\"\":\"\"Bus...|          \"business\"|\n",
            "|\"http://www.midwe...|\"3651\"|\"{\"\"title\"\":\"\"A T...|            \"sports\"|\n",
            "|\"http://www.ivill...|\"1917\"|\"{\"\"title\"\":\"\"9 F...|            \"health\"|\n",
            "|\"http://www.theda...|\"1063\"|\"{\"\"title\"\":\"\" \"\"...|                 \"?\"|\n",
            "|\"http://www.phill...|\"2110\"|\"{\"\"url\"\":\"\"phill...|        \"recreation\"|\n",
            "|\"http://sportsill...|\"5318\"|\"{\"\"title\"\":\"\"Iza...|        \"recreation\"|\n",
            "|\"http://theawesom...|\"9842\"|\"{\"\"title\"\":\"\"Liq...|        \"recreation\"|\n",
            "|\"http://www.guard...|\"3946\"|\"{\"\"title\"\":null,...|\"arts_entertainment\"|\n",
            "|\"http://sugarcraf...| \"334\"|\"{\"\"title\"\":\"\"Gri...|        \"recreation\"|\n",
            "|\"http://www.ivill...| \"271\"|\"{\"\"title\"\":\"\"How...|            \"health\"|\n",
            "+--------------------+------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data_nlp.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wh5Bixk9OYUj"
      },
      "source": [
        "**Cleaning data_nlp**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "84f690c1-cc25-4047-8591-e72f4c5a2eed",
        "id": "ESCqpHo-OYUj"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------+-----------------+------------------+\n",
            "|     Column_Name|Null_Values_Count|Null_Value_Percent|\n",
            "+----------------+-----------------+------------------+\n",
            "|             url|                0|               0.0|\n",
            "|           urlid|                0|               0.0|\n",
            "|     boilerplate|                0|               0.0|\n",
            "|alchemy_category|                0|               0.0|\n",
            "+----------------+-----------------+------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.sql.functions import *\n",
        "\n",
        "def null_value_calc(df):\n",
        "    null_columns_counts = []\n",
        "    numRows = df.count()\n",
        "    for k in df.columns:\n",
        "        nullRows = df.where(col(k).isNull()).count()\n",
        "        temp = k,nullRows,(nullRows/numRows)*100\n",
        "        null_columns_counts.append(temp)\n",
        "    return(null_columns_counts)\n",
        "\n",
        "null_columns_calc_list = null_value_calc(data_nlp)\n",
        "spark.createDataFrame(null_columns_calc_list, ['Column_Name', 'Null_Values_Count','Null_Value_Percent']).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SIWz6Kg1vDow"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.feature import * #CountVectorizer,StringIndexer, RegexTokenizer,StopWordsRemover\n",
        "from pyspark.sql.functions import * #col, udf,regexp_replace,isnull\n",
        "from pyspark.sql.types import * #StringType,IntegerType\n",
        "from pyspark.ml.classification import *\n",
        "from pyspark.ml.evaluation import *\n",
        "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
        "\n",
        "# For pipeline development\n",
        "from pyspark.ml import Pipeline "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GdRAk0Upn5YY",
        "outputId": "45353259-e140-40fd-c11c-f3e38696b736"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------------------------------------------------------------------------------------------------------+-----+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+\n",
            "|url                                                                                                                   |urlid|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |alchemy_category  |\n",
            "+----------------------------------------------------------------------------------------------------------------------+-----+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+\n",
            "|http://www.bloomberg.com/news/2010-12-23/ibm-predicts-holographic-calls-air-breathing-batteries-by-2015.html          |4042 |{title:IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries,body:A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\u2019s what international business machines corp. sees in its crystal ball.,url:bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |business          |\n",
            "|http://www.popsci.com/technology/article/2012-07/electronic-futuristic-starting-gun-eliminates-advantages-races       |8471 |{title:The Fully Electronic Futuristic Starting Gun That Eliminates Advantages in Races the fully electronic, futuristic starting gun that eliminates advantages in races the fully electronic, futuristic starting gun that eliminates advantages in races,body:And that can be carried on a plane without the hassle too The Omega E Gun Starting Pistol Omega It s easy to take for granted just how insanely close some Olympic races are and how much the minutiae of it all can matter The perfect example is the traditional starting gun Seems easy You pull a trigger and the race starts Boom What people don t consider When a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others That s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise Omega has been the watch company tasked as the official timekeeper of the Olympic Games since 1932 At the 2010 Vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling About as far away as you can get probably while still clearly being a starting gun Pull the trigger once and off the Olympians go If it s pressed twice consecutively it signals a false start Working through a speaker system is what eliminates any kind of advantage for athletes It s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win Powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them This year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time It wouldn t be an enormous difference Omega Timing board member Peter H\\u00fcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete They all hear the start commands and signal at exactly the same moment There s also an ulterior reason for its look In a post September 11th world a gun on its way to a major event is going to raise more than a few TSA eyebrows even if it s a realistic looking fake Rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun But there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some Pavlovian response after more than a century of Olympic games that make people want to hear the real thing not a whiny electronic noise Everyone in the stands at home thankfully will still be getting that The sound is programmable and can be synthesized to sound like almost anything H\\u00fcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle, too technology,gadgets,london 2012,london olympics,olympics,omega,starting guns,summer olympics,timing,popular science,popsci,url:popsci technology article 2012 07 electronic futuristic starting gun eliminates advantages races}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |recreation        |\n",
            "|http://www.menshealth.com/health/flu-fighting-fruits?cm_mmc=Facebook-_-MensHealth-_-Content-Health-_-FightFluWithFruit|1164 |{title:Fruits that Fight the Flu fruits that fight the flu | cold & flu | men's health,body:Apples The most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to 1 500 mg of vitamin C Apples are loaded with protective flavonoids which may prevent heart disease and cancer Next Papayas With 250 percent of the RDA of vitamin C a papaya can help kick a cold right out of your system The beta carotene and vitamins C and E in papayas reduce inflammation throughout the body lessening the effects of asthma Next Cranberries Cranberries have more antioxidants than other common fruits and veggies One serving has five times the amount in broccoli Cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses Next Grapefruit Loaded with vitamin C grapefruit also contains natural compounds called limonoids which can lower cholesterol The red varieties are a potent source of the cancer fighting substance lycopene Next Bananas One of the top food sources of vitamin B6 bananas help reduce fatigue depression stress and insomnia Bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure Next everything you need to know about cold and flu so you don\\u2019t get sick this season, at men\\u2019s health.com cold, flu, infection, sore throat, sneeze, immunity, germs, allergies, stay healthy, sick, contagious, medicines, cold medicine,url:menshealth health flu fighting fruits cm mmc Facebook Mens Health Content Health Fight Flu With Fruit}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |health            |\n",
            "|http://www.dumblittleman.com/2007/12/10-foolproof-tips-for-better-sleep.html                                          |6684 |{title:10 Foolproof Tips for Better Sleep ,body:There was a period in my life when I had a lot of problems with sleep It took me very long to fall asleep I was easily awaken and I simply wasn t getting enough of rest at night I didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia Some of these tips I try to follow regularly Don t worry about not getting enough sleep Try not to worry about how much you sleep Such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia Learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia Don t force yourself to sleep The very attempt of trying to do so actually awakes you making it more difficult to sleep Go to bed only when you are feeling really tired and sleepy Don t look at the alarm clock at night Looking at the clock promotes increased anxiety and obsession about time Body heating procedures Some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep Avoid oversleep Don t oversleep to make up for a poor night s sleep Doing so for even a couple of days can reset your body clock and make it harder for you to sleep at night Sex Sex is a well known nighttime stress reliever Healthy sex life enhances your relationship relaxes your body releases happy chemicals and even promotes wellness And it welcomes sleep Avoid alcohol as a sleeping aid Avoid the use of alcohol in the late evening The most common myth found among people is that they believe alcohol helps in the sleep But the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run Associate your bed and bedroom with sleep and sex only Don t watch TV eat or read in bed Although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake Naps If you suffer from insomnia try not taking a nap If the goal is to sleep more during the night napping may steal hours desired later on If you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens Written by C Simmons of HealthAssist net dumb little man shares ideas to make the everyday person more productive in life. expect to read tips on finance, saving money, business, and some diy for the house. tips,diy,money,finance,advice,productivity,efficient,technology,saving,software,business,tools,url:dumblittleman 2007 12 10 foolproof tips for better sleep html}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |health            |\n",
            "|http://bleacherreport.com/articles/1205138-the-50-coolest-jerseys-you-didnt-know-existed?show_full=                   |9006 |{title:The 50 Coolest Jerseys You Didn t Know Existed coolest jerseys you haven't seen,body:Jersey sales is a curious business Whether you re buying the stylish top to represent your favorite team player or color you re always missing out on better artwork With No 18 Colts jerseys continuing to flood the streets it s about time we educate the sports public about the real masterpieces that have yet to be embraced Forget importance or legacy these upcoming selections will be based solely on visual brilliance We ve scoured the surface of the earth in hopes of finding the ultimate collection of sports jerseys And it s about time we share the findings Here are 50 ridiculous jerseys to brighten your day Grab a camera these shots don t come along often 50 Philadelphia Phillies Throwback Jersey With a curvaceous P on the front and a kelly green backdrop it s time to blow the mothballs off of this legendary masterpiece Solid hard nosed and pure Meet Pete Rose 49 Houston Aeros Away Jersey These designers have thrust their Aeros into the upper tier of the American Hockey League with this curious creation Aggressive yet elegant 48 Florida Panthers 3rd Jersey Blue and darker blue aren t exactly Panthers colors but this jersey as a whole reeks of solid domination On the other hand it also screams Pittsburgh Penguins 47 Pittsburgh Pirates Negro League Jersey There s nothing like Heritage Week to bring back the old days The Homestead Grays clearly believed in simplicity and class Less continues to be more 46 Edmonton Oil Kings Military Jersey This Western Hockey League club is prepared to do whatever it takes to cement its name in history forever And that includes fashioning a Jackson Pollack esque display of paint madness More simply referred to as the Military Appreciation Jerseys Abstract art at its finest 45 Cincinnati Stingers Away Jersey This 70s World Hockey Association masterpiece can t possibly be gulped all at once It s too powerful Here is the rest 44 Kingston Frontenacs Don Cherry Jersey The Kingston Frontenacs are still several suits away from reaching greatness but the Ontario Hockey League just got a lot cooler with this mockery of NHL analyst and nut Don Cherry Risky business though as this can often be misconstrued as brutal 43 Atlanta Braves Throwback Jersey Worn by Hank Aaron and his pack of Braves during the franchise s first two seasons in Atlanta 1966 67 this simple off white piece of cloth is a pleasant vacation for the pupils With a player number replacing the chest high tomahawk and an iconic badge on the sleeve these jerseys present solid compliments to the everyday white and red 42 Montreal Canadiens Throwback Jersey When the Canadiens broke out their 1910 11 Centennial jerseys few were ready for such festive aggression An early Christmas present 41 Missouri Tigers New Jersey Forget the heart of a lion This Missouri squad is ready to prove they possess the mind of a Tiger Enhanced durability makes these cats ferocious 40 North Stars Jersey Minnesota enjoyed the green and yellow North Stars from 1967 to 1993 before they moved to Dallas and became the Stars Minnesota Wild 2000 present brass evidently decided not to follow suit 39 Pittsburgh Steelers Throwback Jersey If these 80th season jerseys truly are inspired by bumblebees then due credit goes to those talented honey dwellers They may be tough on the eyes but these obtuse jerseys are a refreshing and necessary addition to the NFL Only in doses though 38 France New Away Jersey The stripe work may seem excessive but let s be real You can never have enough stripes Just as long as France doesn t get mistaken for a pack of referees 37 Anaheim Ducks Goldberg Jersey David Wright continues to amaze us with his potent disregard for the routine This is no regular Ducks sweater But it s clearly phenomenal 36 Montreal Canadiens Centennial Jersey These 1912 1913 throwbacks give us another reason to love stripes But we re still searching for Waldo 35 Turkey New Away Jersey Simple yet effective is the direction these designers seemed headed for The edgy lettering and gnarly ray of light shining from the back makes this potent presentation almost magical 34 Halifax Mooseheads Jersey The firm red shoulders pads curved lettering and complimentary script are the main ingredients for artistic brilliance The CHL is simply epic 33 California Golden Seals Jersey One of six teams added to the league as part of the 1967 NHL expansion the California Golden Seals never truly grasped the importance of winning on the ice they moved to Cleveland and became the Barons in 1976 They were clearly more focused on keeping their elegant white skates squeaky clean 32 Tampa Bay Lightning 3rd Jersey Released in 08 these silver trimmed knockouts are subtly intense It s refreshing The diagonal style lettering is a slight but necessary detail 31 Miami Heat Orange and Pink Jersey They only played in the ABA for two years 1968 1970 but that didn t keep today s superstars from remembering the Miami Floridians White pink orange black It might be too festive Though never too unique 30 Mighty Ducks of Anaheim 3rd Jersey Not only does the centerpiece bird look like a steroid struck or murderous version of Daffy Duck but he also appears ready to dominate the ice They re far from traditional and far from boring 29 Connecticut Whale Jersey Since 2010 this AHL uniform has dominated the competition The Hartford Whalers esque feel gives the Whale a mighty advantage on camera 28 Los Angeles Kings Throwback Jersey This purple and yellow elegance has rarely let down the illustrious Lakers so perhaps the Kings should turn back the clock and embrace their retro look Beating the Devils for the massive Stanley Cup would be a memorable first step to a new tradition 27 Maryland Terrapins New Jersey If the checkered oddly shaped new Maryland jerseys don t get your senses riled up perhaps their gold alternates will Loving the shoulder pad creativity at this moment although this wild flow may become a visual issue in the future 26 Portugal New Away Jersey The precise coloring cornered numbering and vibrant crosshairs make Portugal s new away kit one for the ages Quality over quantity 25 ZSC Lions Victoria Cup Jersey Founded in 1930 the ZSC Lions are clearly experienced in the art of victory But this Switzerland club proved they were more than just a stylish crew when they beat the Chicago Blackhawks in 2009 for the Victoria Cup trophy 24 Boston Bruins 3rd Jersey With a bear hogging the center there s little about this jersey that doesn t scream amazing We may despise Boston s hockey club but we love their Bruin 23 Washington Redskins Alternate Jersey These yellow and red studs are a refreshing clean cut addition to the often cliched gridiron fashion business Letter circling on the helmet makes this a knockout 22 Rutgers Scarlet Knights New Jersey Underneath this fresh style is an exuberant smile unable to contain itself Oh yes look closely The Scarlet Knights seem ready for battle 21 Montreal Canadiens Centennial Jersey Word has it that this blue masterpiece was only worn for one year 1909 1910 but man what a year it was Time for a comeback 20 England New Away Jersey Rugby is already grueling enough but with a duel kit like this few others have a chance The staredown is classic 19 New York Rangers Heritage Jersey 18 Memphis Grizzlies Throwback Jersey As our eyes steadily adjust to the vibrant presentation it becomes clear that the Memphis Grizzlies have an intricate plan for victory Pleasantly distract all viewers 17 France New Home Jersey A ferocious blue display is perfectly complimented by slight gold lettering on the chest and sleeves Nike s new France home kit has all of Western Europe in a euphoric state 16 San Antonio Spurs Throwback Jersey From 1967 to 1973 the Dallas Chaparrals were easily the most stunning pack of ballers in the ABA And then they moved to San Antonio long before the Spurs decided to pay necessary tribute 15 New York Mets Throwback Jersey It seems the Mets have finally realized the atrocity surrounding their black uniforms A legendary NY scripted on their chest seals the deal Respect for baseball s former New York Giants 14 Quidditch Olympics Jersey The nerdiest sport on the planet just became perhaps the most fashionable as well with the introduction of their 2012 Summer Olympic tops Muggles may not possess the power to soar on broom sticks but they clearly know what it takes to look good 13 Oakland Athletics Throwback Jersey Just picture mustache extraordinaire Rollie Fingers sporting the green and yellow stirrups Truly kicking it old school 12 Chicago Blackhawks Throwback Jersey This 1936 inspired sweater does everything to escape the norm The tribal logo sets the tone while the cream white and red striping puts the finishing touches Epic 11 Texas Rangers Throwback Jersey This 1976 powder blue home uniform rests officially as one of the best to have ever graced the diamond The colorfully striped rims are a nice touch 10 Detroit Red Wings Throwback Jersey During their Cougar days Detroit s beloved franchise fashioned an intricate yet clean arsenal This eye pleasing presentation would define the Original Six and the NHL for years to come 9 Croatia New Home Jersey The design team designated to Croatia s EURO 2012 home kit seems to be simultaneously poking at greatness and insanity But let s be real this is pure genius 8 Ottawa Senators Heritage Jersey It must ve been tough for Senators captain Daniel Alfredsson to contain his exuberance during this moment and we respect him for trying Sure the design is creatively perfect but it s the story behind the cloth that has us in a frenzy All it took was one suggestion from graphic designer Jacob Barrette on an Ottawa Senators message board 7 Chicago White Sox Throwback Jersey Dick Allen s hipster glasses and epic stare hog this intense moment but we re still intrigued by the powder blue button down The Ambrose Burnside sideburns complete this masterful shot 6 Los Angeles Lakers Throwback Jersey The Minneapolis Lakers would only last 13 years 1947 1960 but they certainly cemented their spot in sports lore with five George Mikan led championships and vibrant suits Perfection 5 Great Britain New Olympics Jersey Thank you Leicester Riders forward and national team captain Andrew Sullivan for the pink touch The glistening top reeks of class and the solid neck of aggression Nicely done 4 Cleveland Cavaliers Hardwood Classic Jersey We saw the Cavs sport a blue and orange top during LeBron James days but not so much this retro piece The 1970 s wine and gold look is a keeper 3 Ohio State Buckeyes Throwback Jersey In honor of the 1954 Buckeyes team the Ohio State football team was seen repping these artistic keepers at Michigan Stadium several years back From the intricate sleeve work to the simple yet engaging helmets these uniforms epically compliment the gridiron 2 Golden State Warriors Throwback Jersey A faint blue and red trimming cements this less is more masterpiece in hardwood lore for good Purely classic 1 Brooklyn Dodgers Throwback Jersey What a brilliant shot for all Dodgers fans only in California of course The 1940 s road alternates help these West Coast superstars look the part rankings\\/list, multiple sports, b\\/r swagger, partners, sfgate, latimes, cool lists, outbrain, boston partners jersey sales is a curious business. whether you're buying the stylish top to represent your favorite team, player or color, you're always missing out on better artwork.  with no... jersey sales is a curious business. whether you're buying the stylish top to represent your favorite team, player or color, you're always missing out on better artwork.  with no...,url:bleacherreport articles 1205138 the 50 coolest jerseys you didnt know existed show_full}|sports            |\n",
            "|http://www.conveniencemedical.com/genital-herpes-home.php                                                             |7018 |{url:conveniencemedical genital herpes home php,title:Genital Herpes Treatment ,body:Genital herpes is caused by herpes simplex virus HSV 1 the most common type usually appears as cold sores on the mouth and lips whereas HSV 2 affects the genitals buttocks and rectum Patient s have reported that symptoms can appear five days after coming into contact with the virus on the other hand it may be months or even years until the first outbreak Unfortunately once you have caught the genital herpes virus there is no cure for it You can prevent the virus spreading and manage pain caused by the blisters with a course of anti viral treatment Should I have sex if I have genital herpes When you have symptoms of genital herpes HSV is very contagious when blisters are present and there is a high chance of passing on the virus if you have sex You should not have sex from the time symptoms first start until they are fully over If you do have sex using a condom may not fully protect against passing on the virus as the condom only protects the area that is covered When you do not have symptoms which is most of the time It is less likely that you will pass the virus on when you have sex However some virus will be present on the genital skin surface from time to time although infrequently So there is still a small chance that you may pass on the virus when you have sex when you do not have symptoms People who take antiviral medication long term to prevent recurrences have a reduced risk of passing on the virus What is aciclovir it is an antiviral tablet which aims to reduce the severity and duration of an outbreak of genital herpes How do I take the medicine One tablet three times a day for five days Start taking it as soon as you feel a herpes outbreak beginning What are the potential side effects Like any medication these include nausea vomiting diarrhoea or an allergic rash can occur Should this occur you should stop this medication immediately and contact us via your secure online patient record How effective is this treatment Should be effective in most patients if taken as soon as the signs of an outbreak occur How long will one course of treatment last One course is sufficient for a single outbreak genital herpes is a sexually transmitted infection caused by the herpes simplex virus (hsv), and is transmitted through unprotected and oral sex. anyone who is sexually active can catch genital herpes and if the genital herpes is left untreated it may spread to other parts of the body.}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |?                 |\n",
            "|http://gofashionlane.blogspot.tw/2012/06/american-wild-child.html                                                     |8685 |{title:fashion lane American Wild Child ,body:Our favorite summer holiday is just around the corner and boy do we have a heavy dose of outfit inspiration just waiting for you American Wild Child the newest breed of trend reports features what we know are stellar 4th of July looks This summer holiday is the perfect occasion to rock all things Americana influenced whether it be a straight up American Flag over sized scarf styling note If you are beach pool bound this beauty makes for an amazing wrap cover up Winning a Furst of a Kind American flag Denim vest a red white and blue tie dye tank or even our favorite shoe freaks the Damsel bootie in the American flag print No matter what you are doing you are sure going to find a rad outfit that screams I am proud to be an American All pieces in the Trend Report are available at an LF near you PH Daniel Kincaid Styling Alexandra Sherman MU Hair Kait Marie Model Beth Whitson Vision ,url:gofashionlane blogspot tw 2012 06 american wild child html}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |arts_entertainment|\n",
            "|http://www.insidershealth.com/article/racing_for_recovery/3471                                                        |3402 |{url:insidershealth article racing for recovery 3471,title:Racing For Recovery by Dean Johnson racing for recovery by dean johnson - - insidershealth.com,body:Racing For Recovery is the growing idea that drug addiction and alcoholism can be cured by focusing one s energies towards something positive Eddie Freas tell CNN that I feel better when I m working out It does wonders for the mind The reason I started running it was a switch that went off in my head I started feeling positive and feeling great about myself It is true that dopamine levels can be increased by strenuous physical activity but is simply getting your fix in a healthy way actually healthy for you in the long run Freas admits to trying Alcoholics Anonymous and rehab but says the institutions did nothing to cure his drug addiction perhaps that is because of AA s non belief in the ability of a true addict to get cured they look at it as a disease that like cancer can only go into remission In any case Freas is sober today and has gotten there by doing something he loves that much is commendable Whether or not this strategy will better his quality of life or keep him sober over time has yet to be seen The trick apparently isn t just to exercise but to find something anything that will help This Content is restricted to our Community Login or Sign Up for an account to continue reading an insidershealth.com article regarding - racing for recovery by dean johnson. racing for recovery is the growing idea that drug addiction and alcoholism can be cured by focusing one&rsquo;s energies towards something positive. , , alternative health, article, homeopathic, health, natural medicine, natural cures, community}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |?                 |\n",
            "+----------------------------------------------------------------------------------------------------------------------+-----+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------+\n",
            "only showing top 8 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Remove \"\"\n",
        "df = data_nlp.withColumn(\"alchemy_category\",regexp_replace(col('alchemy_category'), '\"', \"\"))\n",
        "df1 = df.withColumn(\"url\",regexp_replace(col('url'), '\"', \"\"))\n",
        "df3 = df1.withColumn(\"urlid\",regexp_replace(col('urlid'), '\"', \"\"))\n",
        "df4 = df3.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), '\"', \"\"))\n",
        "\n",
        "\n",
        "\n",
        "df4.show(8,False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JIFqCsdmyedm"
      },
      "outputs": [],
      "source": [
        "data_nlp=df4"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ezRZKEOoyo_l"
      },
      "source": [
        "**we got urlid so I think it's better to delete the column url since we have a column that refers to it**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yPEZd4QZzApO",
        "outputId": "e6f758ad-2802-4a3c-b8b0-51e052176777"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+------------------+\n",
            "|urlid|         boilerplate|  alchemy_category|\n",
            "+-----+--------------------+------------------+\n",
            "| 4042|{title:IBM Sees H...|          business|\n",
            "| 8471|{title:The Fully ...|        recreation|\n",
            "| 1164|{title:Fruits tha...|            health|\n",
            "| 6684|{title:10 Foolpro...|            health|\n",
            "| 9006|{title:The 50 Coo...|            sports|\n",
            "| 7018|{url:conveniencem...|                 ?|\n",
            "| 8685|{title:fashion la...|arts_entertainment|\n",
            "| 3402|{url:insidersheal...|                 ?|\n",
            "|  477|{title:Valet The ...|                 ?|\n",
            "| 6731|{url:howsweeteats...|                 ?|\n",
            "| 2293|{title:Business F...|          business|\n",
            "| 3651|{title:A Tip of t...|            sports|\n",
            "| 1917|{title:9 Foods Th...|            health|\n",
            "| 1063|{title: ,body: ,u...|                 ?|\n",
            "| 2110|{url:phillyburbs ...|        recreation|\n",
            "| 5318|{title:Izabel Gou...|        recreation|\n",
            "| 9842|{title:Liquid Mou...|        recreation|\n",
            "| 3946|{title:null,body:...|arts_entertainment|\n",
            "|  334|{title:Grilled Pe...|        recreation|\n",
            "|  271|{title:How to Mak...|            health|\n",
            "+-----+--------------------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data_nlp=data_nlp.drop('url')\n",
        "data_nlp.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D36XmbHWzhc2"
      },
      "source": [
        " **alchemy_category is clearly a categorical column so let's apply one-hot encoding to it! In this case, I think there is no need to replace '?' with a specific value since the column is gonna be replaced with a new column.** "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x7IR1yIp0OHT",
        "outputId": "f3c7882d-df44-46e8-9e39-a151087b4055"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+------------------+--------+\n",
            "|urlid|         boilerplate|  alchemy_category|ac_Index|\n",
            "+-----+--------------------+------------------+--------+\n",
            "| 4042|{title:IBM Sees H...|          business|     3.0|\n",
            "| 8471|{title:The Fully ...|        recreation|     1.0|\n",
            "| 1164|{title:Fruits tha...|            health|     4.0|\n",
            "| 6684|{title:10 Foolpro...|            health|     4.0|\n",
            "| 9006|{title:The 50 Coo...|            sports|     5.0|\n",
            "| 7018|{url:conveniencem...|                 ?|     0.0|\n",
            "| 8685|{title:fashion la...|arts_entertainment|     2.0|\n",
            "| 3402|{url:insidersheal...|                 ?|     0.0|\n",
            "|  477|{title:Valet The ...|                 ?|     0.0|\n",
            "| 6731|{url:howsweeteats...|                 ?|     0.0|\n",
            "| 2293|{title:Business F...|          business|     3.0|\n",
            "| 3651|{title:A Tip of t...|            sports|     5.0|\n",
            "| 1917|{title:9 Foods Th...|            health|     4.0|\n",
            "| 1063|{title: ,body: ,u...|                 ?|     0.0|\n",
            "| 2110|{url:phillyburbs ...|        recreation|     1.0|\n",
            "| 5318|{title:Izabel Gou...|        recreation|     1.0|\n",
            "| 9842|{title:Liquid Mou...|        recreation|     1.0|\n",
            "| 3946|{title:null,body:...|arts_entertainment|     2.0|\n",
            "|  334|{title:Grilled Pe...|        recreation|     1.0|\n",
            "|  271|{title:How to Mak...|            health|     4.0|\n",
            "+-----+--------------------+------------------+--------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from pyspark.ml.feature import StringIndexer\n",
        "ac_indexer = StringIndexer(inputCol=\"alchemy_category\", outputCol=\"ac_Index\")\n",
        "#Fits a model to the input dataset with optional parameters.\n",
        "data_nlp = ac_indexer.fit(data_nlp).transform(data_nlp)\n",
        "data_nlp.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fKBEU8Z1c47"
      },
      "source": [
        "Now, we can delete alchemy_category"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QqbCmRT41hbl",
        "outputId": "63e4ade0-0421-4dcd-fc77-103cb28ce47e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+--------+\n",
            "|urlid|         boilerplate|ac_Index|\n",
            "+-----+--------------------+--------+\n",
            "| 4042|{title:IBM Sees H...|     3.0|\n",
            "| 8471|{title:The Fully ...|     1.0|\n",
            "| 1164|{title:Fruits tha...|     4.0|\n",
            "| 6684|{title:10 Foolpro...|     4.0|\n",
            "| 9006|{title:The 50 Coo...|     5.0|\n",
            "| 7018|{url:conveniencem...|     0.0|\n",
            "| 8685|{title:fashion la...|     2.0|\n",
            "| 3402|{url:insidersheal...|     0.0|\n",
            "|  477|{title:Valet The ...|     0.0|\n",
            "| 6731|{url:howsweeteats...|     0.0|\n",
            "| 2293|{title:Business F...|     3.0|\n",
            "| 3651|{title:A Tip of t...|     5.0|\n",
            "| 1917|{title:9 Foods Th...|     4.0|\n",
            "| 1063|{title: ,body: ,u...|     0.0|\n",
            "| 2110|{url:phillyburbs ...|     1.0|\n",
            "| 5318|{title:Izabel Gou...|     1.0|\n",
            "| 9842|{title:Liquid Mou...|     1.0|\n",
            "| 3946|{title:null,body:...|     2.0|\n",
            "|  334|{title:Grilled Pe...|     1.0|\n",
            "|  271|{title:How to Mak...|     4.0|\n",
            "+-----+--------------------+--------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data_nlp=data_nlp.drop('alchemy_category')\n",
        "data_nlp.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u3VtmGp61v-A"
      },
      "source": [
        "**Clean the boilerplate column**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bK9D_Jhr11mx",
        "outputId": "9992a865-4e83-4c72-ed5d-a13803dbc751"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "| title IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls, air-breathing batteries,body A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By 2015 your mobile phone will project a 3 D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its 3 000 researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested 5 8 billion in research and development last year 6 1 percent of revenue While that s down from about 10 percent in the early 1990s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent 2 4 percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs 500 miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose 9 cents to 146 04 at 11 02 a m in New York Stock Exchange composite trading The stock had gained 11 percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last 10 times longer in 2015 than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of 2006 researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more 2 D environments of Facebook Inc and Twitter Inc Meanwhile a 2007 prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles5 bloomberg net by 2015, your mobile phone will project a 3-d image of anyone who calls and your laptop will be powered by kinetic energy. at least that\\u2019s what international business machines corp. sees in its crystal ball.,url bloomberg news 2010 12 23 ibm predicts holographic calls air breathing batteries by 2015 html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |\n",
            "| title The Fully Electronic Futuristic Starting Gun That Eliminates Advantages in Races the fully electronic, futuristic starting gun that eliminates advantages in races the fully electronic, futuristic starting gun that eliminates advantages in races,body And that can be carried on a plane without the hassle too The Omega E Gun Starting Pistol Omega It s easy to take for granted just how insanely close some Olympic races are and how much the minutiae of it all can matter The perfect example is the traditional starting gun Seems easy You pull a trigger and the race starts Boom What people don t consider When a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others That s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise Omega has been the watch company tasked as the official timekeeper of the Olympic Games since 1932 At the 2010 Vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling About as far away as you can get probably while still clearly being a starting gun Pull the trigger once and off the Olympians go If it s pressed twice consecutively it signals a false start Working through a speaker system is what eliminates any kind of advantage for athletes It s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win Powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them This year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time It wouldn t be an enormous difference Omega Timing board member Peter H\\u00fcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete They all hear the start commands and signal at exactly the same moment There s also an ulterior reason for its look In a post September 11th world a gun on its way to a major event is going to raise more than a few TSA eyebrows even if it s a realistic looking fake Rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun But there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some Pavlovian response after more than a century of Olympic games that make people want to hear the real thing not a whiny electronic noise Everyone in the stands at home thankfully will still be getting that The sound is programmable and can be synthesized to sound like almost anything H\\u00fcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle, too technology,gadgets,london 2012,london olympics,olympics,omega,starting guns,summer olympics,timing,popular science,popsci,url popsci technology article 2012 07 electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |\n",
            "| title Fruits that Fight the Flu fruits that fight the flu | cold & flu | men's health,body Apples The most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to 1 500 mg of vitamin C Apples are loaded with protective flavonoids which may prevent heart disease and cancer Next Papayas With 250 percent of the RDA of vitamin C a papaya can help kick a cold right out of your system The beta carotene and vitamins C and E in papayas reduce inflammation throughout the body lessening the effects of asthma Next Cranberries Cranberries have more antioxidants than other common fruits and veggies One serving has five times the amount in broccoli Cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses Next Grapefruit Loaded with vitamin C grapefruit also contains natural compounds called limonoids which can lower cholesterol The red varieties are a potent source of the cancer fighting substance lycopene Next Bananas One of the top food sources of vitamin B6 bananas help reduce fatigue depression stress and insomnia Bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure Next everything you need to know about cold and flu so you don\\u2019t get sick this season, at men\\u2019s health.com cold, flu, infection, sore throat, sneeze, immunity, germs, allergies, stay healthy, sick, contagious, medicines, cold medicine,url menshealth health flu fighting fruits cm mmc Facebook Mens Health Content Health Fight Flu With Fruit                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
            "| title 10 Foolproof Tips for Better Sleep ,body There was a period in my life when I had a lot of problems with sleep It took me very long to fall asleep I was easily awaken and I simply wasn t getting enough of rest at night I didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia Some of these tips I try to follow regularly Don t worry about not getting enough sleep Try not to worry about how much you sleep Such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia Learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia Don t force yourself to sleep The very attempt of trying to do so actually awakes you making it more difficult to sleep Go to bed only when you are feeling really tired and sleepy Don t look at the alarm clock at night Looking at the clock promotes increased anxiety and obsession about time Body heating procedures Some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep Avoid oversleep Don t oversleep to make up for a poor night s sleep Doing so for even a couple of days can reset your body clock and make it harder for you to sleep at night Sex Sex is a well known nighttime stress reliever Healthy sex life enhances your relationship relaxes your body releases happy chemicals and even promotes wellness And it welcomes sleep Avoid alcohol as a sleeping aid Avoid the use of alcohol in the late evening The most common myth found among people is that they believe alcohol helps in the sleep But the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run Associate your bed and bedroom with sleep and sex only Don t watch TV eat or read in bed Although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake Naps If you suffer from insomnia try not taking a nap If the goal is to sleep more during the night napping may steal hours desired later on If you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens Written by C Simmons of HealthAssist net dumb little man shares ideas to make the everyday person more productive in life. expect to read tips on finance, saving money, business, and some diy for the house. tips,diy,money,finance,advice,productivity,efficient,technology,saving,software,business,tools,url dumblittleman 2007 12 10 foolproof tips for better sleep html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |\n",
            "| title The 50 Coolest Jerseys You Didn t Know Existed coolest jerseys you haven't seen,body Jersey sales is a curious business Whether you re buying the stylish top to represent your favorite team player or color you re always missing out on better artwork With No 18 Colts jerseys continuing to flood the streets it s about time we educate the sports public about the real masterpieces that have yet to be embraced Forget importance or legacy these upcoming selections will be based solely on visual brilliance We ve scoured the surface of the earth in hopes of finding the ultimate collection of sports jerseys And it s about time we share the findings Here are 50 ridiculous jerseys to brighten your day Grab a camera these shots don t come along often 50 Philadelphia Phillies Throwback Jersey With a curvaceous P on the front and a kelly green backdrop it s time to blow the mothballs off of this legendary masterpiece Solid hard nosed and pure Meet Pete Rose 49 Houston Aeros Away Jersey These designers have thrust their Aeros into the upper tier of the American Hockey League with this curious creation Aggressive yet elegant 48 Florida Panthers 3rd Jersey Blue and darker blue aren t exactly Panthers colors but this jersey as a whole reeks of solid domination On the other hand it also screams Pittsburgh Penguins 47 Pittsburgh Pirates Negro League Jersey There s nothing like Heritage Week to bring back the old days The Homestead Grays clearly believed in simplicity and class Less continues to be more 46 Edmonton Oil Kings Military Jersey This Western Hockey League club is prepared to do whatever it takes to cement its name in history forever And that includes fashioning a Jackson Pollack esque display of paint madness More simply referred to as the Military Appreciation Jerseys Abstract art at its finest 45 Cincinnati Stingers Away Jersey This 70s World Hockey Association masterpiece can t possibly be gulped all at once It s too powerful Here is the rest 44 Kingston Frontenacs Don Cherry Jersey The Kingston Frontenacs are still several suits away from reaching greatness but the Ontario Hockey League just got a lot cooler with this mockery of NHL analyst and nut Don Cherry Risky business though as this can often be misconstrued as brutal 43 Atlanta Braves Throwback Jersey Worn by Hank Aaron and his pack of Braves during the franchise s first two seasons in Atlanta 1966 67 this simple off white piece of cloth is a pleasant vacation for the pupils With a player number replacing the chest high tomahawk and an iconic badge on the sleeve these jerseys present solid compliments to the everyday white and red 42 Montreal Canadiens Throwback Jersey When the Canadiens broke out their 1910 11 Centennial jerseys few were ready for such festive aggression An early Christmas present 41 Missouri Tigers New Jersey Forget the heart of a lion This Missouri squad is ready to prove they possess the mind of a Tiger Enhanced durability makes these cats ferocious 40 North Stars Jersey Minnesota enjoyed the green and yellow North Stars from 1967 to 1993 before they moved to Dallas and became the Stars Minnesota Wild 2000 present brass evidently decided not to follow suit 39 Pittsburgh Steelers Throwback Jersey If these 80th season jerseys truly are inspired by bumblebees then due credit goes to those talented honey dwellers They may be tough on the eyes but these obtuse jerseys are a refreshing and necessary addition to the NFL Only in doses though 38 France New Away Jersey The stripe work may seem excessive but let s be real You can never have enough stripes Just as long as France doesn t get mistaken for a pack of referees 37 Anaheim Ducks Goldberg Jersey David Wright continues to amaze us with his potent disregard for the routine This is no regular Ducks sweater But it s clearly phenomenal 36 Montreal Canadiens Centennial Jersey These 1912 1913 throwbacks give us another reason to love stripes But we re still searching for Waldo 35 Turkey New Away Jersey Simple yet effective is the direction these designers seemed headed for The edgy lettering and gnarly ray of light shining from the back makes this potent presentation almost magical 34 Halifax Mooseheads Jersey The firm red shoulders pads curved lettering and complimentary script are the main ingredients for artistic brilliance The CHL is simply epic 33 California Golden Seals Jersey One of six teams added to the league as part of the 1967 NHL expansion the California Golden Seals never truly grasped the importance of winning on the ice they moved to Cleveland and became the Barons in 1976 They were clearly more focused on keeping their elegant white skates squeaky clean 32 Tampa Bay Lightning 3rd Jersey Released in 08 these silver trimmed knockouts are subtly intense It s refreshing The diagonal style lettering is a slight but necessary detail 31 Miami Heat Orange and Pink Jersey They only played in the ABA for two years 1968 1970 but that didn t keep today s superstars from remembering the Miami Floridians White pink orange black It might be too festive Though never too unique 30 Mighty Ducks of Anaheim 3rd Jersey Not only does the centerpiece bird look like a steroid struck or murderous version of Daffy Duck but he also appears ready to dominate the ice They re far from traditional and far from boring 29 Connecticut Whale Jersey Since 2010 this AHL uniform has dominated the competition The Hartford Whalers esque feel gives the Whale a mighty advantage on camera 28 Los Angeles Kings Throwback Jersey This purple and yellow elegance has rarely let down the illustrious Lakers so perhaps the Kings should turn back the clock and embrace their retro look Beating the Devils for the massive Stanley Cup would be a memorable first step to a new tradition 27 Maryland Terrapins New Jersey If the checkered oddly shaped new Maryland jerseys don t get your senses riled up perhaps their gold alternates will Loving the shoulder pad creativity at this moment although this wild flow may become a visual issue in the future 26 Portugal New Away Jersey The precise coloring cornered numbering and vibrant crosshairs make Portugal s new away kit one for the ages Quality over quantity 25 ZSC Lions Victoria Cup Jersey Founded in 1930 the ZSC Lions are clearly experienced in the art of victory But this Switzerland club proved they were more than just a stylish crew when they beat the Chicago Blackhawks in 2009 for the Victoria Cup trophy 24 Boston Bruins 3rd Jersey With a bear hogging the center there s little about this jersey that doesn t scream amazing We may despise Boston s hockey club but we love their Bruin 23 Washington Redskins Alternate Jersey These yellow and red studs are a refreshing clean cut addition to the often cliched gridiron fashion business Letter circling on the helmet makes this a knockout 22 Rutgers Scarlet Knights New Jersey Underneath this fresh style is an exuberant smile unable to contain itself Oh yes look closely The Scarlet Knights seem ready for battle 21 Montreal Canadiens Centennial Jersey Word has it that this blue masterpiece was only worn for one year 1909 1910 but man what a year it was Time for a comeback 20 England New Away Jersey Rugby is already grueling enough but with a duel kit like this few others have a chance The staredown is classic 19 New York Rangers Heritage Jersey 18 Memphis Grizzlies Throwback Jersey As our eyes steadily adjust to the vibrant presentation it becomes clear that the Memphis Grizzlies have an intricate plan for victory Pleasantly distract all viewers 17 France New Home Jersey A ferocious blue display is perfectly complimented by slight gold lettering on the chest and sleeves Nike s new France home kit has all of Western Europe in a euphoric state 16 San Antonio Spurs Throwback Jersey From 1967 to 1973 the Dallas Chaparrals were easily the most stunning pack of ballers in the ABA And then they moved to San Antonio long before the Spurs decided to pay necessary tribute 15 New York Mets Throwback Jersey It seems the Mets have finally realized the atrocity surrounding their black uniforms A legendary NY scripted on their chest seals the deal Respect for baseball s former New York Giants 14 Quidditch Olympics Jersey The nerdiest sport on the planet just became perhaps the most fashionable as well with the introduction of their 2012 Summer Olympic tops Muggles may not possess the power to soar on broom sticks but they clearly know what it takes to look good 13 Oakland Athletics Throwback Jersey Just picture mustache extraordinaire Rollie Fingers sporting the green and yellow stirrups Truly kicking it old school 12 Chicago Blackhawks Throwback Jersey This 1936 inspired sweater does everything to escape the norm The tribal logo sets the tone while the cream white and red striping puts the finishing touches Epic 11 Texas Rangers Throwback Jersey This 1976 powder blue home uniform rests officially as one of the best to have ever graced the diamond The colorfully striped rims are a nice touch 10 Detroit Red Wings Throwback Jersey During their Cougar days Detroit s beloved franchise fashioned an intricate yet clean arsenal This eye pleasing presentation would define the Original Six and the NHL for years to come 9 Croatia New Home Jersey The design team designated to Croatia s EURO 2012 home kit seems to be simultaneously poking at greatness and insanity But let s be real this is pure genius 8 Ottawa Senators Heritage Jersey It must ve been tough for Senators captain Daniel Alfredsson to contain his exuberance during this moment and we respect him for trying Sure the design is creatively perfect but it s the story behind the cloth that has us in a frenzy All it took was one suggestion from graphic designer Jacob Barrette on an Ottawa Senators message board 7 Chicago White Sox Throwback Jersey Dick Allen s hipster glasses and epic stare hog this intense moment but we re still intrigued by the powder blue button down The Ambrose Burnside sideburns complete this masterful shot 6 Los Angeles Lakers Throwback Jersey The Minneapolis Lakers would only last 13 years 1947 1960 but they certainly cemented their spot in sports lore with five George Mikan led championships and vibrant suits Perfection 5 Great Britain New Olympics Jersey Thank you Leicester Riders forward and national team captain Andrew Sullivan for the pink touch The glistening top reeks of class and the solid neck of aggression Nicely done 4 Cleveland Cavaliers Hardwood Classic Jersey We saw the Cavs sport a blue and orange top during LeBron James days but not so much this retro piece The 1970 s wine and gold look is a keeper 3 Ohio State Buckeyes Throwback Jersey In honor of the 1954 Buckeyes team the Ohio State football team was seen repping these artistic keepers at Michigan Stadium several years back From the intricate sleeve work to the simple yet engaging helmets these uniforms epically compliment the gridiron 2 Golden State Warriors Throwback Jersey A faint blue and red trimming cements this less is more masterpiece in hardwood lore for good Purely classic 1 Brooklyn Dodgers Throwback Jersey What a brilliant shot for all Dodgers fans only in California of course The 1940 s road alternates help these West Coast superstars look the part rankings\\/list, multiple sports, b\\/r swagger, partners, sfgate, latimes, cool lists, outbrain, boston partners jersey sales is a curious business. whether you're buying the stylish top to represent your favorite team, player or color, you're always missing out on better artwork.  with no... jersey sales is a curious business. whether you're buying the stylish top to represent your favorite team, player or color, you're always missing out on better artwork.  with no...,url bleacherreport articles 1205138 the 50 coolest jerseys you didnt know existed show_full |\n",
            "| url conveniencemedical genital herpes home php,title Genital Herpes Treatment ,body Genital herpes is caused by herpes simplex virus HSV 1 the most common type usually appears as cold sores on the mouth and lips whereas HSV 2 affects the genitals buttocks and rectum Patient s have reported that symptoms can appear five days after coming into contact with the virus on the other hand it may be months or even years until the first outbreak Unfortunately once you have caught the genital herpes virus there is no cure for it You can prevent the virus spreading and manage pain caused by the blisters with a course of anti viral treatment Should I have sex if I have genital herpes When you have symptoms of genital herpes HSV is very contagious when blisters are present and there is a high chance of passing on the virus if you have sex You should not have sex from the time symptoms first start until they are fully over If you do have sex using a condom may not fully protect against passing on the virus as the condom only protects the area that is covered When you do not have symptoms which is most of the time It is less likely that you will pass the virus on when you have sex However some virus will be present on the genital skin surface from time to time although infrequently So there is still a small chance that you may pass on the virus when you have sex when you do not have symptoms People who take antiviral medication long term to prevent recurrences have a reduced risk of passing on the virus What is aciclovir it is an antiviral tablet which aims to reduce the severity and duration of an outbreak of genital herpes How do I take the medicine One tablet three times a day for five days Start taking it as soon as you feel a herpes outbreak beginning What are the potential side effects Like any medication these include nausea vomiting diarrhoea or an allergic rash can occur Should this occur you should stop this medication immediately and contact us via your secure online patient record How effective is this treatment Should be effective in most patients if taken as soon as the signs of an outbreak occur How long will one course of treatment last One course is sufficient for a single outbreak genital herpes is a sexually transmitted infection caused by the herpes simplex virus (hsv), and is transmitted through unprotected and oral sex. anyone who is sexually active can catch genital herpes and if the genital herpes is left untreated it may spread to other parts of the body.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "| title fashion lane American Wild Child ,body Our favorite summer holiday is just around the corner and boy do we have a heavy dose of outfit inspiration just waiting for you American Wild Child the newest breed of trend reports features what we know are stellar 4th of July looks This summer holiday is the perfect occasion to rock all things Americana influenced whether it be a straight up American Flag over sized scarf styling note If you are beach pool bound this beauty makes for an amazing wrap cover up Winning a Furst of a Kind American flag Denim vest a red white and blue tie dye tank or even our favorite shoe freaks the Damsel bootie in the American flag print No matter what you are doing you are sure going to find a rad outfit that screams I am proud to be an American All pieces in the Trend Report are available at an LF near you PH Daniel Kincaid Styling Alexandra Sherman MU Hair Kait Marie Model Beth Whitson Vision ,url gofashionlane blogspot tw 2012 06 american wild child html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 7 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Replace Slashes and parenthesis with spaces\n",
        "# You can test your script on line 7 of the df \"(Legend of Zelda/Fable Inspired)\"\n",
        "data_nlp = data_nlp.withColumn(\"boilerplate\",translate(col(\"boilerplate\"), \":\", \" \")) \\\n",
        "        .withColumn(\"boilerplate\",translate(col(\"boilerplate\"), \"{\", \" \")) \\\n",
        "        .withColumn(\"boilerplate\",translate(col(\"boilerplate\"), \"}\", \" \"))\n",
        "data_nlp.select(\"boilerplate\").show(7,False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p4bfrdUr1rKg",
        "outputId": "14c3bebc-05a0-4cfc-82cd-74eae453ff6b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "| title IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls airbreathing batteriesbody A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By  your mobile phone will project a  D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its   researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested   billion in research and development last year   percent of revenue While that s down from about  percent in the early s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent   percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs  miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose  cents to   at   a m in New York Stock Exchange composite trading The stock had gained  percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last  times longer in  than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of  researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more  D environments of Facebook Inc and Twitter Inc Meanwhile a  prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles bloomberg net by  your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least thatus what international business machines corp sees in its crystal ballurl bloomberg news    ibm predicts holographic calls air breathing batteries by  html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
            "| title The Fully Electronic Futuristic Starting Gun That Eliminates Advantages in Races the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in racesbody And that can be carried on a plane without the hassle too The Omega E Gun Starting Pistol Omega It s easy to take for granted just how insanely close some Olympic races are and how much the minutiae of it all can matter The perfect example is the traditional starting gun Seems easy You pull a trigger and the race starts Boom What people don t consider When a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others That s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise Omega has been the watch company tasked as the official timekeeper of the Olympic Games since  At the  Vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling About as far away as you can get probably while still clearly being a starting gun Pull the trigger once and off the Olympians go If it s pressed twice consecutively it signals a false start Working through a speaker system is what eliminates any kind of advantage for athletes It s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win Powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them This year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time It wouldn t be an enormous difference Omega Timing board member Peter Hufcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete They all hear the start commands and signal at exactly the same moment There s also an ulterior reason for its look In a post September th world a gun on its way to a major event is going to raise more than a few TSA eyebrows even if it s a realistic looking fake Rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun But there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some Pavlovian response after more than a century of Olympic games that make people want to hear the real thing not a whiny electronic noise Everyone in the stands at home thankfully will still be getting that The sound is programmable and can be synthesized to sound like almost anything Hufcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle too technologygadgetslondon london olympicsolympicsomegastarting gunssummer olympicstimingpopular sciencepopsciurl popsci technology article   electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "| title Fruits that Fight the Flu fruits that fight the flu  cold  flu  mens healthbody Apples The most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to   mg of vitamin C Apples are loaded with protective flavonoids which may prevent heart disease and cancer Next Papayas With  percent of the RDA of vitamin C a papaya can help kick a cold right out of your system The beta carotene and vitamins C and E in papayas reduce inflammation throughout the body lessening the effects of asthma Next Cranberries Cranberries have more antioxidants than other common fruits and veggies One serving has five times the amount in broccoli Cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses Next Grapefruit Loaded with vitamin C grapefruit also contains natural compounds called limonoids which can lower cholesterol The red varieties are a potent source of the cancer fighting substance lycopene Next Bananas One of the top food sources of vitamin B bananas help reduce fatigue depression stress and insomnia Bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure Next everything you need to know about cold and flu so you donut get sick this season at menus healthcom cold flu infection sore throat sneeze immunity germs allergies stay healthy sick contagious medicines cold medicineurl menshealth health flu fighting fruits cm mmc Facebook Mens Health Content Health Fight Flu With Fruit                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |\n",
            "| title  Foolproof Tips for Better Sleep body There was a period in my life when I had a lot of problems with sleep It took me very long to fall asleep I was easily awaken and I simply wasn t getting enough of rest at night I didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia Some of these tips I try to follow regularly Don t worry about not getting enough sleep Try not to worry about how much you sleep Such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia Learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia Don t force yourself to sleep The very attempt of trying to do so actually awakes you making it more difficult to sleep Go to bed only when you are feeling really tired and sleepy Don t look at the alarm clock at night Looking at the clock promotes increased anxiety and obsession about time Body heating procedures Some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep Avoid oversleep Don t oversleep to make up for a poor night s sleep Doing so for even a couple of days can reset your body clock and make it harder for you to sleep at night Sex Sex is a well known nighttime stress reliever Healthy sex life enhances your relationship relaxes your body releases happy chemicals and even promotes wellness And it welcomes sleep Avoid alcohol as a sleeping aid Avoid the use of alcohol in the late evening The most common myth found among people is that they believe alcohol helps in the sleep But the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run Associate your bed and bedroom with sleep and sex only Don t watch TV eat or read in bed Although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake Naps If you suffer from insomnia try not taking a nap If the goal is to sleep more during the night napping may steal hours desired later on If you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens Written by C Simmons of HealthAssist net dumb little man shares ideas to make the everyday person more productive in life expect to read tips on finance saving money business and some diy for the house tipsdiymoneyfinanceadviceproductivityefficienttechnologysavingsoftwarebusinesstoolsurl dumblittleman    foolproof tips for better sleep html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "| title The  Coolest Jerseys You Didn t Know Existed coolest jerseys you havent seenbody Jersey sales is a curious business Whether you re buying the stylish top to represent your favorite team player or color you re always missing out on better artwork With No  Colts jerseys continuing to flood the streets it s about time we educate the sports public about the real masterpieces that have yet to be embraced Forget importance or legacy these upcoming selections will be based solely on visual brilliance We ve scoured the surface of the earth in hopes of finding the ultimate collection of sports jerseys And it s about time we share the findings Here are  ridiculous jerseys to brighten your day Grab a camera these shots don t come along often  Philadelphia Phillies Throwback Jersey With a curvaceous P on the front and a kelly green backdrop it s time to blow the mothballs off of this legendary masterpiece Solid hard nosed and pure Meet Pete Rose  Houston Aeros Away Jersey These designers have thrust their Aeros into the upper tier of the American Hockey League with this curious creation Aggressive yet elegant  Florida Panthers rd Jersey Blue and darker blue aren t exactly Panthers colors but this jersey as a whole reeks of solid domination On the other hand it also screams Pittsburgh Penguins  Pittsburgh Pirates Negro League Jersey There s nothing like Heritage Week to bring back the old days The Homestead Grays clearly believed in simplicity and class Less continues to be more  Edmonton Oil Kings Military Jersey This Western Hockey League club is prepared to do whatever it takes to cement its name in history forever And that includes fashioning a Jackson Pollack esque display of paint madness More simply referred to as the Military Appreciation Jerseys Abstract art at its finest  Cincinnati Stingers Away Jersey This s World Hockey Association masterpiece can t possibly be gulped all at once It s too powerful Here is the rest  Kingston Frontenacs Don Cherry Jersey The Kingston Frontenacs are still several suits away from reaching greatness but the Ontario Hockey League just got a lot cooler with this mockery of NHL analyst and nut Don Cherry Risky business though as this can often be misconstrued as brutal  Atlanta Braves Throwback Jersey Worn by Hank Aaron and his pack of Braves during the franchise s first two seasons in Atlanta   this simple off white piece of cloth is a pleasant vacation for the pupils With a player number replacing the chest high tomahawk and an iconic badge on the sleeve these jerseys present solid compliments to the everyday white and red  Montreal Canadiens Throwback Jersey When the Canadiens broke out their   Centennial jerseys few were ready for such festive aggression An early Christmas present  Missouri Tigers New Jersey Forget the heart of a lion This Missouri squad is ready to prove they possess the mind of a Tiger Enhanced durability makes these cats ferocious  North Stars Jersey Minnesota enjoyed the green and yellow North Stars from  to  before they moved to Dallas and became the Stars Minnesota Wild  present brass evidently decided not to follow suit  Pittsburgh Steelers Throwback Jersey If these th season jerseys truly are inspired by bumblebees then due credit goes to those talented honey dwellers They may be tough on the eyes but these obtuse jerseys are a refreshing and necessary addition to the NFL Only in doses though  France New Away Jersey The stripe work may seem excessive but let s be real You can never have enough stripes Just as long as France doesn t get mistaken for a pack of referees  Anaheim Ducks Goldberg Jersey David Wright continues to amaze us with his potent disregard for the routine This is no regular Ducks sweater But it s clearly phenomenal  Montreal Canadiens Centennial Jersey These   throwbacks give us another reason to love stripes But we re still searching for Waldo  Turkey New Away Jersey Simple yet effective is the direction these designers seemed headed for The edgy lettering and gnarly ray of light shining from the back makes this potent presentation almost magical  Halifax Mooseheads Jersey The firm red shoulders pads curved lettering and complimentary script are the main ingredients for artistic brilliance The CHL is simply epic  California Golden Seals Jersey One of six teams added to the league as part of the  NHL expansion the California Golden Seals never truly grasped the importance of winning on the ice they moved to Cleveland and became the Barons in  They were clearly more focused on keeping their elegant white skates squeaky clean  Tampa Bay Lightning rd Jersey Released in  these silver trimmed knockouts are subtly intense It s refreshing The diagonal style lettering is a slight but necessary detail  Miami Heat Orange and Pink Jersey They only played in the ABA for two years   but that didn t keep today s superstars from remembering the Miami Floridians White pink orange black It might be too festive Though never too unique  Mighty Ducks of Anaheim rd Jersey Not only does the centerpiece bird look like a steroid struck or murderous version of Daffy Duck but he also appears ready to dominate the ice They re far from traditional and far from boring  Connecticut Whale Jersey Since  this AHL uniform has dominated the competition The Hartford Whalers esque feel gives the Whale a mighty advantage on camera  Los Angeles Kings Throwback Jersey This purple and yellow elegance has rarely let down the illustrious Lakers so perhaps the Kings should turn back the clock and embrace their retro look Beating the Devils for the massive Stanley Cup would be a memorable first step to a new tradition  Maryland Terrapins New Jersey If the checkered oddly shaped new Maryland jerseys don t get your senses riled up perhaps their gold alternates will Loving the shoulder pad creativity at this moment although this wild flow may become a visual issue in the future  Portugal New Away Jersey The precise coloring cornered numbering and vibrant crosshairs make Portugal s new away kit one for the ages Quality over quantity  ZSC Lions Victoria Cup Jersey Founded in  the ZSC Lions are clearly experienced in the art of victory But this Switzerland club proved they were more than just a stylish crew when they beat the Chicago Blackhawks in  for the Victoria Cup trophy  Boston Bruins rd Jersey With a bear hogging the center there s little about this jersey that doesn t scream amazing We may despise Boston s hockey club but we love their Bruin  Washington Redskins Alternate Jersey These yellow and red studs are a refreshing clean cut addition to the often cliched gridiron fashion business Letter circling on the helmet makes this a knockout  Rutgers Scarlet Knights New Jersey Underneath this fresh style is an exuberant smile unable to contain itself Oh yes look closely The Scarlet Knights seem ready for battle  Montreal Canadiens Centennial Jersey Word has it that this blue masterpiece was only worn for one year   but man what a year it was Time for a comeback  England New Away Jersey Rugby is already grueling enough but with a duel kit like this few others have a chance The staredown is classic  New York Rangers Heritage Jersey  Memphis Grizzlies Throwback Jersey As our eyes steadily adjust to the vibrant presentation it becomes clear that the Memphis Grizzlies have an intricate plan for victory Pleasantly distract all viewers  France New Home Jersey A ferocious blue display is perfectly complimented by slight gold lettering on the chest and sleeves Nike s new France home kit has all of Western Europe in a euphoric state  San Antonio Spurs Throwback Jersey From  to  the Dallas Chaparrals were easily the most stunning pack of ballers in the ABA And then they moved to San Antonio long before the Spurs decided to pay necessary tribute  New York Mets Throwback Jersey It seems the Mets have finally realized the atrocity surrounding their black uniforms A legendary NY scripted on their chest seals the deal Respect for baseball s former New York Giants  Quidditch Olympics Jersey The nerdiest sport on the planet just became perhaps the most fashionable as well with the introduction of their  Summer Olympic tops Muggles may not possess the power to soar on broom sticks but they clearly know what it takes to look good  Oakland Athletics Throwback Jersey Just picture mustache extraordinaire Rollie Fingers sporting the green and yellow stirrups Truly kicking it old school  Chicago Blackhawks Throwback Jersey This  inspired sweater does everything to escape the norm The tribal logo sets the tone while the cream white and red striping puts the finishing touches Epic  Texas Rangers Throwback Jersey This  powder blue home uniform rests officially as one of the best to have ever graced the diamond The colorfully striped rims are a nice touch  Detroit Red Wings Throwback Jersey During their Cougar days Detroit s beloved franchise fashioned an intricate yet clean arsenal This eye pleasing presentation would define the Original Six and the NHL for years to come  Croatia New Home Jersey The design team designated to Croatia s EURO  home kit seems to be simultaneously poking at greatness and insanity But let s be real this is pure genius  Ottawa Senators Heritage Jersey It must ve been tough for Senators captain Daniel Alfredsson to contain his exuberance during this moment and we respect him for trying Sure the design is creatively perfect but it s the story behind the cloth that has us in a frenzy All it took was one suggestion from graphic designer Jacob Barrette on an Ottawa Senators message board  Chicago White Sox Throwback Jersey Dick Allen s hipster glasses and epic stare hog this intense moment but we re still intrigued by the powder blue button down The Ambrose Burnside sideburns complete this masterful shot  Los Angeles Lakers Throwback Jersey The Minneapolis Lakers would only last  years   but they certainly cemented their spot in sports lore with five George Mikan led championships and vibrant suits Perfection  Great Britain New Olympics Jersey Thank you Leicester Riders forward and national team captain Andrew Sullivan for the pink touch The glistening top reeks of class and the solid neck of aggression Nicely done  Cleveland Cavaliers Hardwood Classic Jersey We saw the Cavs sport a blue and orange top during LeBron James days but not so much this retro piece The  s wine and gold look is a keeper  Ohio State Buckeyes Throwback Jersey In honor of the  Buckeyes team the Ohio State football team was seen repping these artistic keepers at Michigan Stadium several years back From the intricate sleeve work to the simple yet engaging helmets these uniforms epically compliment the gridiron  Golden State Warriors Throwback Jersey A faint blue and red trimming cements this less is more masterpiece in hardwood lore for good Purely classic  Brooklyn Dodgers Throwback Jersey What a brilliant shot for all Dodgers fans only in California of course The  s road alternates help these West Coast superstars look the part rankingslist multiple sports br swagger partners sfgate latimes cool lists outbrain boston partners jersey sales is a curious business whether youre buying the stylish top to represent your favorite team player or color youre always missing out on better artwork  with no jersey sales is a curious business whether youre buying the stylish top to represent your favorite team player or color youre always missing out on better artwork  with nourl bleacherreport articles  the  coolest jerseys you didnt know existed showfull |\n",
            "| url conveniencemedical genital herpes home phptitle Genital Herpes Treatment body Genital herpes is caused by herpes simplex virus HSV  the most common type usually appears as cold sores on the mouth and lips whereas HSV  affects the genitals buttocks and rectum Patient s have reported that symptoms can appear five days after coming into contact with the virus on the other hand it may be months or even years until the first outbreak Unfortunately once you have caught the genital herpes virus there is no cure for it You can prevent the virus spreading and manage pain caused by the blisters with a course of anti viral treatment Should I have sex if I have genital herpes When you have symptoms of genital herpes HSV is very contagious when blisters are present and there is a high chance of passing on the virus if you have sex You should not have sex from the time symptoms first start until they are fully over If you do have sex using a condom may not fully protect against passing on the virus as the condom only protects the area that is covered When you do not have symptoms which is most of the time It is less likely that you will pass the virus on when you have sex However some virus will be present on the genital skin surface from time to time although infrequently So there is still a small chance that you may pass on the virus when you have sex when you do not have symptoms People who take antiviral medication long term to prevent recurrences have a reduced risk of passing on the virus What is aciclovir it is an antiviral tablet which aims to reduce the severity and duration of an outbreak of genital herpes How do I take the medicine One tablet three times a day for five days Start taking it as soon as you feel a herpes outbreak beginning What are the potential side effects Like any medication these include nausea vomiting diarrhoea or an allergic rash can occur Should this occur you should stop this medication immediately and contact us via your secure online patient record How effective is this treatment Should be effective in most patients if taken as soon as the signs of an outbreak occur How long will one course of treatment last One course is sufficient for a single outbreak genital herpes is a sexually transmitted infection caused by the herpes simplex virus hsv and is transmitted through unprotected and oral sex anyone who is sexually active can catch genital herpes and if the genital herpes is left untreated it may spread to other parts of the body                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |\n",
            "| title fashion lane American Wild Child body Our favorite summer holiday is just around the corner and boy do we have a heavy dose of outfit inspiration just waiting for you American Wild Child the newest breed of trend reports features what we know are stellar th of July looks This summer holiday is the perfect occasion to rock all things Americana influenced whether it be a straight up American Flag over sized scarf styling note If you are beach pool bound this beauty makes for an amazing wrap cover up Winning a Furst of a Kind American flag Denim vest a red white and blue tie dye tank or even our favorite shoe freaks the Damsel bootie in the American flag print No matter what you are doing you are sure going to find a rad outfit that screams I am proud to be an American All pieces in the Trend Report are available at an LF near you PH Daniel Kincaid Styling Alexandra Sherman MU Hair Kait Marie Model Beth Whitson Vision url gofashionlane blogspot tw   american wild child html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "| url insidershealth article racing for recovery title Racing For Recovery by Dean Johnson racing for recovery by dean johnson   insidershealthcombody Racing For Recovery is the growing idea that drug addiction and alcoholism can be cured by focusing one s energies towards something positive Eddie Freas tell CNN that I feel better when I m working out It does wonders for the mind The reason I started running it was a switch that went off in my head I started feeling positive and feeling great about myself It is true that dopamine levels can be increased by strenuous physical activity but is simply getting your fix in a healthy way actually healthy for you in the long run Freas admits to trying Alcoholics Anonymous and rehab but says the institutions did nothing to cure his drug addiction perhaps that is because of AA s non belief in the ability of a true addict to get cured they look at it as a disease that like cancer can only go into remission In any case Freas is sober today and has gotten there by doing something he loves that much is commendable Whether or not this strategy will better his quality of life or keep him sober over time has yet to be seen The trick apparently isn t just to exercise but to find something anything that will help This Content is restricted to our Community Login or Sign Up for an account to continue reading an insidershealthcom article regarding  racing for recovery by dean johnson racing for recovery is the growing idea that drug addiction and alcoholism can be cured by focusing onersquos energies towards something positive   alternative health article homeopathic health natural medicine natural cures community                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "| title Valet The Handbook  Days  daysbody Resolutions are for suckers Instead of swearing off sugar for all of two weeks simply vow to live better For the fourth annual  Days we re offering you all the tips and tricks needed to shape up for  Each day brings new expert advice on how to look your best and live life to the fullest resolutions are for suckers instead of swearing off sugar for all of two weeks simply vow to live better for the fourth annual  days were offering you all the tips and tricks needed to shape up for  each day brings new expert advice on how to look your best live life to the fullest and have a little fun while doing it heres to starting the new decade off in style valet valetmag  days how to tips and tricks style how tos resolutions are for suckers instead of swearing off sugar for all of two weeks simply vow to live better we show you howurl valetmag the handbook features  days index php index middle feature                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "| url howsweeteats    cookies and cream browniestitle Cookies and Cream Brownies How Sweet It Is body More brownies It seems that I can t get through one full week without trying a new brownie combination Just wait until you see the next one I am going to experiment with These are super simple I used my go to brownie recipe and just added a few Oreos You could even do the same with boxed brownies Just make the frosting and add some more cookies I only frosted a few because I just don t love frosting on brownies Mr How Sweet really enjoyed the frosted ones If you waste a good   of your day on Twitter like I do you will know that last night Mr How Sweet brought me home a dozen hot pink roses I didn t make him dinner my half unpacked mess is strewn in  different rooms and I was in holey sweat pants once he got home Oh yes and I was half asleep while he professed his undying love to me And I just continued to lay there Sounds eerily similar to the night he proposed Good thing there were brownies on the counter As my dad says marriage is rarely   The cookies somewhat melted into the batter once baked but if you look closely you can see some cookies inside the brownies That is the best part The frosting is the same frosting I used for the cookies and cream cupcakes It is creamy delicious and looks like ice cream Can t beat that I think out of all the brownies I have made in the past few months these ones may be the best I love the texture Chewy gooey brownies crunchy cookies Cookies and Cream Brownies   squares baking chocolate   lb butter  cups sugar  eggs  cup flour  teaspoon vanilla  Oreo cookies crumbled Preheat oven to  Melt butter and baking sqaures in a microwave safe bowl In the mixing bowl cream melted butter and chocolate mixture Add in sugar and vanilla and beat until fluffy about   minutes Add in eggs beating one at a time Add flour and stir just until combined Fold in cookie crumbles Pour into greased  x  baking dish I used a  x  square cake pan Bake for   minutes or until just barely done Cool before frosting I changed this recipe slightly from my other brownie posts so if you have recently tried it try adding  more eggs Cookies and Cream Frosting  sticks butter softened     lbs powdered sugar  teaspoon vanilla   tablespoons milk  Oreo cookies Mix butter and powdered sugar together adding sugar gradually to the mix Add vanilla Mix in milk adding more if needed Continue to add milk and sugar until it reaches the desired consistency Crush Oreos and fold them into the frosting I don t think these brownies will last another evening in our house Ha who am I kidding They may not even last until Mr How Sweet gets home Now I need to find some spectacular recipe to make for dinner tonight And remind Mr How Sweet that he married someone  years younger not an old granny Tags brownies chocolate recipes more brownies it seems that i cant get through one full week without trying a new brownie combination just wait until you see the next one i am going to brownieschocolaterecipes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Removing anything that is not a letter\n",
        "df = data_nlp.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), '[^A-Za-z ]+', ''))\n",
        "df.select(\"boilerplate\").show(10,False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_EdOYOmCyiZ-",
        "outputId": "a420e500-f4f4-42e8-cd41-17102334a984"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|  IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls airbreathing batteries A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By  your mobile phone will project a  D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its   researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested   billion in research and development last year   percent of revenue While that s down from about  percent in the early s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent   percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs  miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose  cents to   at   a m in New York Stock Exchange composite trading The stock had gained  percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last  times longer in  than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of  researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more  D environments of Facebook Inc and Twitter Inc Meanwhile a  prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles bloomberg net by  your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least thatus what international business machines corp sees in its crystal ball bloomberg news    ibm predicts holographic calls air breathing batteries by  html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |\n",
            "|  The Fully Electronic Futuristic Starting Gun That Eliminates Advantages in Races the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races And that can be carried on a plane without the hassle too The Omega E Gun Starting Pistol Omega It s easy to take for granted just how insanely close some Olympic races are and how much the minutiae of it all can matter The perfect example is the traditional starting gun Seems easy You pull a trigger and the race starts Boom What people don t consider When a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others That s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise Omega has been the watch company tasked as the official timekeeper of the Olympic Games since  At the  Vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling About as far away as you can get probably while still clearly being a starting gun Pull the trigger once and off the Olympians go If it s pressed twice consecutively it signals a false start Working through a speaker system is what eliminates any kind of advantage for athletes It s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win Powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them This year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time It wouldn t be an enormous difference Omega Timing board member Peter Hufcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete They all hear the start commands and signal at exactly the same moment There s also an ulterior reason for its look In a post September th world a gun on its way to a major event is going to raise more than a few TSA eyebrows even if it s a realistic looking fake Rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun But there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some Pavlovian response after more than a century of Olympic games that make people want to hear the real thing not a whiny electronic noise Everyone in the stands at home thankfully will still be getting that The sound is programmable and can be synthesized to sound like almost anything Hufcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle too technologygadgetslondon london olympicsolympicsomegastarting gunssummer olympicstimingpopular sciencepopsci popsci technology article   electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |\n",
            "|  Fruits that Fight the Flu fruits that fight the flu  cold  flu  mens health Apples The most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to   mg of vitamin C Apples are loaded with protective flavonoids which may prevent heart disease and cancer Next Papayas With  percent of the RDA of vitamin C a papaya can help kick a cold right out of your system The beta carotene and vitamins C and E in papayas reduce inflammation throughout the  lessening the effects of asthma Next Cranberries Cranberries have more antioxidants than other common fruits and veggies One serving has five times the amount in broccoli Cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses Next Grapefruit Loaded with vitamin C grapefruit also contains natural compounds called limonoids which can lower cholesterol The red varieties are a potent source of the cancer fighting substance lycopene Next Bananas One of the top food sources of vitamin B bananas help reduce fatigue depression stress and insomnia Bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure Next everything you need to know about cold and flu so you donut get sick this season at menus healthcom cold flu infection sore throat sneeze immunity germs allergies stay healthy sick contagious medicines cold medicine menshealth health flu fighting fruits cm mmc Facebook Mens Health Content Health Fight Flu With Fruit                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |\n",
            "|   Foolproof Tips for Better Sleep  There was a period in my life when I had a lot of problems with sleep It took me very long to fall asleep I was easily awaken and I simply wasn t getting enough of rest at night I didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia Some of these tips I try to follow regularly Don t worry about not getting enough sleep Try not to worry about how much you sleep Such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia Learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia Don t force yourself to sleep The very attempt of trying to do so actually awakes you making it more difficult to sleep Go to bed only when you are feeling really tired and sleepy Don t look at the alarm clock at night Looking at the clock promotes increased anxiety and obsession about time Body heating procedures Some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep Avoid oversleep Don t oversleep to make up for a poor night s sleep Doing so for even a couple of days can reset your  clock and make it harder for you to sleep at night Sex Sex is a well known nighttime stress reliever Healthy sex life enhances your relationship relaxes your  releases happy chemicals and even promotes wellness And it welcomes sleep Avoid alcohol as a sleeping aid Avoid the use of alcohol in the late evening The most common myth found among people is that they believe alcohol helps in the sleep But the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run Associate your bed and bedroom with sleep and sex only Don t watch TV eat or read in bed Although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake Naps If you suffer from insomnia try not taking a nap If the goal is to sleep more during the night napping may steal hours desired later on If you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens Written by C Simmons of HealthAssist net dumb little man shares ideas to make the everyday person more productive in life expect to read tips on finance saving money business and some diy for the house tipsdiymoneyfinanceadviceproductivityefficienttechnologysavingsoftwarebusinesstools dumblittleman    foolproof tips for better sleep html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|  The  Coolest Jerseys You Didn t Know Existed coolest jerseys you havent seen Jersey sales is a curious business Whether you re buying the stylish top to represent your favorite team player or color you re always missing out on better artwork With No  Colts jerseys continuing to flood the streets it s about time we educate the sports public about the real masterpieces that have yet to be embraced Forget importance or legacy these upcoming selections will be based solely on visual brilliance We ve scoured the surface of the earth in hopes of finding the ultimate collection of sports jerseys And it s about time we share the findings Here are  ridiculous jerseys to brighten your day Grab a camera these shots don t come along often  Philadelphia Phillies Throwback Jersey With a curvaceous P on the front and a kelly green backdrop it s time to blow the mothballs off of this legendary masterpiece Solid hard nosed and pure Meet Pete Rose  Houston Aeros Away Jersey These designers have thrust their Aeros into the upper tier of the American Hockey League with this curious creation Aggressive yet elegant  Florida Panthers rd Jersey Blue and darker blue aren t exactly Panthers colors but this jersey as a whole reeks of solid domination On the other hand it also screams Pittsburgh Penguins  Pittsburgh Pirates Negro League Jersey There s nothing like Heritage Week to bring back the old days The Homestead Grays clearly believed in simplicity and class Less continues to be more  Edmonton Oil Kings Military Jersey This Western Hockey League club is prepared to do whatever it takes to cement its name in history forever And that includes fashioning a Jackson Pollack esque display of paint madness More simply referred to as the Military Appreciation Jerseys Abstract art at its finest  Cincinnati Stingers Away Jersey This s World Hockey Association masterpiece can t possibly be gulped all at once It s too powerful Here is the rest  Kingston Frontenacs Don Cherry Jersey The Kingston Frontenacs are still several suits away from reaching greatness but the Ontario Hockey League just got a lot cooler with this mockery of NHL analyst and nut Don Cherry Risky business though as this can often be misconstrued as brutal  Atlanta Braves Throwback Jersey Worn by Hank Aaron and his pack of Braves during the franchise s first two seasons in Atlanta   this simple off white piece of cloth is a pleasant vacation for the pupils With a player number replacing the chest high tomahawk and an iconic badge on the sleeve these jerseys present solid compliments to the everyday white and red  Montreal Canadiens Throwback Jersey When the Canadiens broke out their   Centennial jerseys few were ready for such festive aggression An early Christmas present  Missouri Tigers New Jersey Forget the heart of a lion This Missouri squad is ready to prove they possess the mind of a Tiger Enhanced durability makes these cats ferocious  North Stars Jersey Minnesota enjoyed the green and yellow North Stars from  to  before they moved to Dallas and became the Stars Minnesota Wild  present brass evidently decided not to follow suit  Pittsburgh Steelers Throwback Jersey If these th season jerseys truly are inspired by bumblebees then due credit goes to those talented honey dwellers They may be tough on the eyes but these obtuse jerseys are a refreshing and necessary addition to the NFL Only in doses though  France New Away Jersey The stripe work may seem excessive but let s be real You can never have enough stripes Just as long as France doesn t get mistaken for a pack of referees  Anaheim Ducks Goldberg Jersey David Wright continues to amaze us with his potent disregard for the routine This is no regular Ducks sweater But it s clearly phenomenal  Montreal Canadiens Centennial Jersey These   throwbacks give us another reason to love stripes But we re still searching for Waldo  Turkey New Away Jersey Simple yet effective is the direction these designers seemed headed for The edgy lettering and gnarly ray of light shining from the back makes this potent presentation almost magical  Halifax Mooseheads Jersey The firm red shoulders pads curved lettering and complimentary script are the main ingredients for artistic brilliance The CHL is simply epic  California Golden Seals Jersey One of six teams added to the league as part of the  NHL expansion the California Golden Seals never truly grasped the importance of winning on the ice they moved to Cleveland and became the Barons in  They were clearly more focused on keeping their elegant white skates squeaky clean  Tampa Bay Lightning rd Jersey Released in  these silver trimmed knockouts are subtly intense It s refreshing The diagonal style lettering is a slight but necessary detail  Miami Heat Orange and Pink Jersey They only played in the ABA for two years   but that didn t keep today s superstars from remembering the Miami Floridians White pink orange black It might be too festive Though never too unique  Mighty Ducks of Anaheim rd Jersey Not only does the centerpiece bird look like a steroid struck or murderous version of Daffy Duck but he also appears ready to dominate the ice They re far from traditional and far from boring  Connecticut Whale Jersey Since  this AHL uniform has dominated the competition The Hartford Whalers esque feel gives the Whale a mighty advantage on camera  Los Angeles Kings Throwback Jersey This purple and yellow elegance has rarely let down the illustrious Lakers so perhaps the Kings should turn back the clock and embrace their retro look Beating the Devils for the massive Stanley Cup would be a memorable first step to a new tradition  Maryland Terrapins New Jersey If the checkered oddly shaped new Maryland jerseys don t get your senses riled up perhaps their gold alternates will Loving the shoulder pad creativity at this moment although this wild flow may become a visual issue in the future  Portugal New Away Jersey The precise coloring cornered numbering and vibrant crosshairs make Portugal s new away kit one for the ages Quality over quantity  ZSC Lions Victoria Cup Jersey Founded in  the ZSC Lions are clearly experienced in the art of victory But this Switzerland club proved they were more than just a stylish crew when they beat the Chicago Blackhawks in  for the Victoria Cup trophy  Boston Bruins rd Jersey With a bear hogging the center there s little about this jersey that doesn t scream amazing We may despise Boston s hockey club but we love their Bruin  Washington Redskins Alternate Jersey These yellow and red studs are a refreshing clean cut addition to the often cliched gridiron fashion business Letter circling on the helmet makes this a knockout  Rutgers Scarlet Knights New Jersey Underneath this fresh style is an exuberant smile unable to contain itself Oh yes look closely The Scarlet Knights seem ready for battle  Montreal Canadiens Centennial Jersey Word has it that this blue masterpiece was only worn for one year   but man what a year it was Time for a comeback  England New Away Jersey Rugby is already grueling enough but with a duel kit like this few others have a chance The staredown is classic  New York Rangers Heritage Jersey  Memphis Grizzlies Throwback Jersey As our eyes steadily adjust to the vibrant presentation it becomes clear that the Memphis Grizzlies have an intricate plan for victory Pleasantly distract all viewers  France New Home Jersey A ferocious blue display is perfectly complimented by slight gold lettering on the chest and sleeves Nike s new France home kit has all of Western Europe in a euphoric state  San Antonio Spurs Throwback Jersey From  to  the Dallas Chaparrals were easily the most stunning pack of ballers in the ABA And then they moved to San Antonio long before the Spurs decided to pay necessary tribute  New York Mets Throwback Jersey It seems the Mets have finally realized the atrocity surrounding their black uniforms A legendary NY scripted on their chest seals the deal Respect for baseball s former New York Giants  Quidditch Olympics Jersey The nerdiest sport on the planet just became perhaps the most fashionable as well with the introduction of their  Summer Olympic tops Muggles may not possess the power to soar on broom sticks but they clearly know what it takes to look good  Oakland Athletics Throwback Jersey Just picture mustache extraordinaire Rollie Fingers sporting the green and yellow stirrups Truly kicking it old school  Chicago Blackhawks Throwback Jersey This  inspired sweater does everything to escape the norm The tribal logo sets the tone while the cream white and red striping puts the finishing touches Epic  Texas Rangers Throwback Jersey This  powder blue home uniform rests officially as one of the best to have ever graced the diamond The colorfully striped rims are a nice touch  Detroit Red Wings Throwback Jersey During their Cougar days Detroit s beloved franchise fashioned an intricate yet clean arsenal This eye pleasing presentation would define the Original Six and the NHL for years to come  Croatia New Home Jersey The design team designated to Croatia s EURO  home kit seems to be simultaneously poking at greatness and insanity But let s be real this is pure genius  Ottawa Senators Heritage Jersey It must ve been tough for Senators captain Daniel Alfredsson to contain his exuberance during this moment and we respect him for trying Sure the design is creatively perfect but it s the story behind the cloth that has us in a frenzy All it took was one suggestion from graphic designer Jacob Barrette on an Ottawa Senators message board  Chicago White Sox Throwback Jersey Dick Allen s hipster glasses and epic stare hog this intense moment but we re still intrigued by the powder blue button down The Ambrose Burnside sideburns complete this masterful shot  Los Angeles Lakers Throwback Jersey The Minneapolis Lakers would only last  years   but they certainly cemented their spot in sports lore with five George Mikan led championships and vibrant suits Perfection  Great Britain New Olympics Jersey Thank you Leicester Riders forward and national team captain Andrew Sullivan for the pink touch The glistening top reeks of class and the solid neck of aggression Nicely done  Cleveland Cavaliers Hardwood Classic Jersey We saw the Cavs sport a blue and orange top during LeBron James days but not so much this retro piece The  s wine and gold look is a keeper  Ohio State Buckeyes Throwback Jersey In honor of the  Buckeyes team the Ohio State football team was seen repping these artistic keepers at Michigan Stadium several years back From the intricate sleeve work to the simple yet engaging helmets these uniforms epically compliment the gridiron  Golden State Warriors Throwback Jersey A faint blue and red trimming cements this less is more masterpiece in hardwood lore for good Purely classic  Brooklyn Dodgers Throwback Jersey What a brilliant shot for all Dodgers fans only in California of course The  s road alternates help these West Coast superstars look the part rankingslist multiple sports br swagger partners sfgate latimes cool lists outbrain boston partners jersey sales is a curious business whether youre buying the stylish top to represent your favorite team player or color youre always missing out on better artwork  with no jersey sales is a curious business whether youre buying the stylish top to represent your favorite team player or color youre always missing out on better artwork  with no bleacherreport articles  the  coolest jerseys you didnt know existed showfull |\n",
            "|  conveniencemedical genital herpes home php Genital Herpes Treatment  Genital herpes is caused by herpes simplex virus HSV  the most common type usually appears as cold sores on the mouth and lips whereas HSV  affects the genitals buttocks and rectum Patient s have reported that symptoms can appear five days after coming into contact with the virus on the other hand it may be months or even years until the first outbreak Unfortunately once you have caught the genital herpes virus there is no cure for it You can prevent the virus spreading and manage pain caused by the blisters with a course of anti viral treatment Should I have sex if I have genital herpes When you have symptoms of genital herpes HSV is very contagious when blisters are present and there is a high chance of passing on the virus if you have sex You should not have sex from the time symptoms first start until they are fully over If you do have sex using a condom may not fully protect against passing on the virus as the condom only protects the area that is covered When you do not have symptoms which is most of the time It is less likely that you will pass the virus on when you have sex However some virus will be present on the genital skin surface from time to time although infrequently So there is still a small chance that you may pass on the virus when you have sex when you do not have symptoms People who take antiviral medication long term to prevent recurrences have a reduced risk of passing on the virus What is aciclovir it is an antiviral tablet which aims to reduce the severity and duration of an outbreak of genital herpes How do I take the medicine One tablet three times a day for five days Start taking it as soon as you feel a herpes outbreak beginning What are the potential side effects Like any medication these include nausea vomiting diarrhoea or an allergic rash can occur Should this occur you should stop this medication immediately and contact us via your secure online patient record How effective is this treatment Should be effective in most patients if taken as soon as the signs of an outbreak occur How long will one course of treatment last One course is sufficient for a single outbreak genital herpes is a sexually transmitted infection caused by the herpes simplex virus hsv and is transmitted through unprotected and oral sex anyone who is sexually active can catch genital herpes and if the genital herpes is left untreated it may spread to other parts of the                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|  fashion lane American Wild Child  Our favorite summer holiday is just around the corner and boy do we have a heavy dose of outfit inspiration just waiting for you American Wild Child the newest breed of trend reports features what we know are stellar th of July looks This summer holiday is the perfect occasion to rock all things Americana influenced whether it be a straight up American Flag over sized scarf styling note If you are beach pool bound this beauty makes for an amazing wrap cover up Winning a Furst of a Kind American flag Denim vest a red white and blue tie dye tank or even our favorite shoe freaks the Damsel bootie in the American flag print No matter what you are doing you are sure going to find a rad outfit that screams I am proud to be an American All pieces in the Trend Report are available at an LF near you PH Daniel Kincaid Styling Alexandra Sherman MU Hair Kait Marie Model Beth Whitson Vision  gofashionlane blogspot tw   american wild child html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "|  insidershealth article racing for recovery  Racing For Recovery by Dean Johnson racing for recovery by dean johnson   insidershealthcom Racing For Recovery is the growing idea that drug addiction and alcoholism can be cured by focusing one s energies towards something positive Eddie Freas tell CNN that I feel better when I m working out It does wonders for the mind The reason I started running it was a switch that went off in my head I started feeling positive and feeling great about myself It is true that dopamine levels can be increased by strenuous physical activity but is simply getting your fix in a healthy way actually healthy for you in the long run Freas admits to trying Alcoholics Anonymous and rehab but says the institutions did nothing to cure his drug addiction perhaps that is because of AA s non belief in the ability of a true addict to get cured they look at it as a disease that like cancer can only go into remission In any case Freas is sober today and has gotten there by doing something he loves that much is commendable Whether or not this strategy will better his quality of life or keep him sober over time has yet to be seen The trick apparently isn t just to exercise but to find something anything that will help This Content is restricted to our Community Login or Sign Up for an account to continue reading an insidershealthcom article regarding  racing for recovery by dean johnson racing for recovery is the growing idea that drug addiction and alcoholism can be cured by focusing onersquos energies towards something positive   alternative health article homeopathic health natural medicine natural cures community                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |\n",
            "|  Valet The Handbook  Days  days Resolutions are for suckers Instead of swearing off sugar for all of two weeks simply vow to live better For the fourth annual  Days we re offering you all the tips and tricks needed to shape up for  Each day brings new expert advice on how to look your best and live life to the fullest resolutions are for suckers instead of swearing off sugar for all of two weeks simply vow to live better for the fourth annual  days were offering you all the tips and tricks needed to shape up for  each day brings new expert advice on how to look your best live life to the fullest and have a little fun while doing it heres to starting the new decade off in style valet valetmag  days how to tips and tricks style how tos resolutions are for suckers instead of swearing off sugar for all of two weeks simply vow to live better we show you how valetmag the handbook features  days index php index middle feature                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "|  howsweeteats    cookies and cream brownies Cookies and Cream Brownies How Sweet It Is  More brownies It seems that I can t get through one full week without trying a new brownie combination Just wait until you see the next one I am going to experiment with These are super simple I used my go to brownie recipe and just added a few Oreos You could even do the same with boxed brownies Just make the frosting and add some more cookies I only frosted a few because I just don t love frosting on brownies Mr How Sweet really enjoyed the frosted ones If you waste a good   of your day on Twitter like I do you will know that last night Mr How Sweet brought me home a dozen hot pink roses I didn t make him dinner my half unpacked mess is strewn in  different rooms and I was in holey sweat pants once he got home Oh yes and I was half asleep while he professed his undying love to me And I just continued to lay there Sounds eerily similar to the night he proposed Good thing there were brownies on the counter As my dad says marriage is rarely   The cookies somewhat melted into the batter once baked but if you look closely you can see some cookies inside the brownies That is the best part The frosting is the same frosting I used for the cookies and cream cupcakes It is creamy delicious and looks like ice cream Can t beat that I think out of all the brownies I have made in the past few months these ones may be the best I love the texture Chewy gooey brownies crunchy cookies Cookies and Cream Brownies   squares baking chocolate   lb butter  cups sugar  eggs  cup flour  teaspoon vanilla  Oreo cookies crumbled Preheat oven to  Melt butter and baking sqaures in a microwave safe bowl In the mixing bowl cream melted butter and chocolate mixture Add in sugar and vanilla and beat until fluffy about   minutes Add in eggs beating one at a time Add flour and stir just until combined Fold in cookie crumbles Pour into greased  x  baking dish I used a  x  square cake pan Bake for   minutes or until just barely done Cool before frosting I changed this recipe slightly from my other brownie posts so if you have recently tried it try adding  more eggs Cookies and Cream Frosting  sticks butter softened     lbs powdered sugar  teaspoon vanilla   tablespoons milk  Oreo cookies Mix butter and powdered sugar together adding sugar gradually to the mix Add vanilla Mix in milk adding more if needed Continue to add milk and sugar until it reaches the desired consistency Crush Oreos and fold them into the frosting I don t think these brownies will last another evening in our house Ha who am I kidding They may not even last until Mr How Sweet gets home Now I need to find some spectacular recipe to make for dinner tonight And remind Mr How Sweet that he married someone  years younger not an old granny Tags brownies chocolate recipes more brownies it seems that i cant get through one full week without trying a new brownie combination just wait until you see the next one i am going to brownieschocolaterecipes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "df = df.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), 'title', ''))\n",
        "df = df.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), 'url', ''))\n",
        "data_nlp = df.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), 'body', ''))\n",
        "data_nlp.select(\"boilerplate\").show(10,False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U2fyWji1vfZe",
        "outputId": "9b4ef06d-1261-42f6-e73a-350261bc6d03"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "| IBM Sees Holographic Calls Air Breathing Batteries ibm sees holographic calls airbreathing batteries A sign stands outside the International Business Machines Corp IBM Almaden Research Center campus in San Jose California Photographer Tony Avelar Bloomberg Buildings stand at the International Business Machines Corp IBM Almaden Research Center campus in the Santa Teresa Hills of San Jose California Photographer Tony Avelar Bloomberg By your mobile phone will project a D image of anyone who calls and your laptop will be powered by kinetic energy At least that s what International Business Machines Corp sees in its crystal ball The predictions are part of an annual tradition for the Armonk New York based company which surveys its researchers to find five ideas expected to take root in the next five years IBM the world s largest provider of computer services looks to Silicon Valley for input gleaning many ideas from its Almaden research center in San Jose California Holographic conversations projected from mobile phones lead this year s list The predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers These are all stretch goals and that s good said Paul Saffo managing director of foresight at the investment advisory firm Discern in San Francisco In an era when pessimism is the new black a little dose of technological optimism is not a bad thing For IBM it s not just idle speculation The company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth Saffo said Not all of its predictions pan out though IBM was overly optimistic about the spread of speech technology for instance When the ideas do lead to products they can have broad implications for society as well as IBM s bottom line he said Research Spending They have continued to do research when all the other grand research organizations are gone said Saffo who is also a consulting associate professor at Stanford University IBM invested billion in research and development last year percent of revenue While that s down from about percent in the early s the company spends a bigger share on research than its computing rivals Hewlett Packard Co the top maker of personal computers spent percent last year At Almaden scientists work on projects that don t always fit in with IBM s computer business The lab s research includes efforts to develop an electric car battery that runs miles on one charge a filtration system for desalination and a program that shows changes in geographic data IBM rose cents to at a m in New York Stock Exchange composite trading The stock had gained percent this year before today Citizen Science The list is meant to give a window into the company s innovation engine said Josephine Cheng a vice president at IBM s Almaden lab All this demonstrates a real culture of innovation at IBM and willingness to devote itself to solving some of the world s biggest problems she said Many of the predictions are based on projects that IBM has in the works One of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative Earlier this year IBM teamed up with the California State Water Resources Control Board and the City of San Jose Environmental Services to help gather information about waterways Researchers from Almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions The hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves Traffic Predictors IBM also sees data helping shorten commutes in the next five years Computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck Batteries may last times longer in than today IBM says Rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge Some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge The final prediction involves recycling the heat generated by computers and data centers Almost half of the power used by data centers is currently spent keeping the computers cool IBM scientists say it would be better to harness that heat to warm houses and offices In IBM s first list of predictions compiled at the end of researchers said instantaneous speech translation would become the norm That hasn t happened yet While some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in Star Trek Second Life The company also predicted that online immersive environments such as Second Life would become more widespread While immersive video games are as popular as ever Second Life s growth has slowed Internet users are flocking instead to the more D environments of Facebook Inc and Twitter Inc Meanwhile a prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications Consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones The nice thing about the list is that it provokes thought Saffo said If everything came true they wouldn t be doing their job To contact the reporter on this story Ryan Flinn in San Francisco at rflinn bloomberg net To contact the editor responsible for this story Tom Giles at tgiles bloomberg net by your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least thatus what international business machines corp sees in its crystal ball bloomberg news ibm predicts holographic calls air breathing batteries by html |\n",
            "| The Fully Electronic Futuristic Starting Gun That Eliminates Advantages in Races the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races And that can be carried on a plane without the hassle too The Omega E Gun Starting Pistol Omega It s easy to take for granted just how insanely close some Olympic races are and how much the minutiae of it all can matter The perfect example is the traditional starting gun Seems easy You pull a trigger and the race starts Boom What people don t consider When a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others That s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise Omega has been the watch company tasked as the official timekeeper of the Olympic Games since At the Vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling About as far away as you can get probably while still clearly being a starting gun Pull the trigger once and off the Olympians go If it s pressed twice consecutively it signals a false start Working through a speaker system is what eliminates any kind of advantage for athletes It s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win Powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them This year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time It wouldn t be an enormous difference Omega Timing board member Peter Hufcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete They all hear the start commands and signal at exactly the same moment There s also an ulterior reason for its look In a post September th world a gun on its way to a major event is going to raise more than a few TSA eyebrows even if it s a realistic looking fake Rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun But there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some Pavlovian response after more than a century of Olympic games that make people want to hear the real thing not a whiny electronic noise Everyone in the stands at home thankfully will still be getting that The sound is programmable and can be synthesized to sound like almost anything Hufcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle too technologygadgetslondon london olympicsolympicsomegastarting gunssummer olympicstimingpopular sciencepopsci popsci technology article electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "| Fruits that Fight the Flu fruits that fight the flu cold flu mens health Apples The most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to mg of vitamin C Apples are loaded with protective flavonoids which may prevent heart disease and cancer Next Papayas With percent of the RDA of vitamin C a papaya can help kick a cold right out of your system The beta carotene and vitamins C and E in papayas reduce inflammation throughout the lessening the effects of asthma Next Cranberries Cranberries have more antioxidants than other common fruits and veggies One serving has five times the amount in broccoli Cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses Next Grapefruit Loaded with vitamin C grapefruit also contains natural compounds called limonoids which can lower cholesterol The red varieties are a potent source of the cancer fighting substance lycopene Next Bananas One of the top food sources of vitamin B bananas help reduce fatigue depression stress and insomnia Bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure Next everything you need to know about cold and flu so you donut get sick this season at menus healthcom cold flu infection sore throat sneeze immunity germs allergies stay healthy sick contagious medicines cold medicine menshealth health flu fighting fruits cm mmc Facebook Mens Health Content Health Fight Flu With Fruit                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "| Foolproof Tips for Better Sleep There was a period in my life when I had a lot of problems with sleep It took me very long to fall asleep I was easily awaken and I simply wasn t getting enough of rest at night I didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia Some of these tips I try to follow regularly Don t worry about not getting enough sleep Try not to worry about how much you sleep Such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia Learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia Don t force yourself to sleep The very attempt of trying to do so actually awakes you making it more difficult to sleep Go to bed only when you are feeling really tired and sleepy Don t look at the alarm clock at night Looking at the clock promotes increased anxiety and obsession about time Body heating procedures Some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep Avoid oversleep Don t oversleep to make up for a poor night s sleep Doing so for even a couple of days can reset your clock and make it harder for you to sleep at night Sex Sex is a well known nighttime stress reliever Healthy sex life enhances your relationship relaxes your releases happy chemicals and even promotes wellness And it welcomes sleep Avoid alcohol as a sleeping aid Avoid the use of alcohol in the late evening The most common myth found among people is that they believe alcohol helps in the sleep But the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run Associate your bed and bedroom with sleep and sex only Don t watch TV eat or read in bed Although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake Naps If you suffer from insomnia try not taking a nap If the goal is to sleep more during the night napping may steal hours desired later on If you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens Written by C Simmons of HealthAssist net dumb little man shares ideas to make the everyday person more productive in life expect to read tips on finance saving money business and some diy for the house tipsdiymoneyfinanceadviceproductivityefficienttechnologysavingsoftwarebusinesstools dumblittleman foolproof tips for better sleep html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Remove multiple spaces\n",
        "data_nlp = data_nlp.withColumn(\"boilerplate\",regexp_replace(col('boilerplate'), ' +', ' '))\n",
        "data_nlp.select(\"boilerplate\").show(4,False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wrlfqkjVPWsM",
        "outputId": "a6b39ea7-7154-4f96-dfea-211f68eb43dd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |\n",
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "| ibm sees holographic calls air breathing batteries ibm sees holographic calls airbreathing batteries a sign stands outside the international business machines corp ibm almaden research center campus in san jose california photographer tony avelar bloomberg buildings stand at the international business machines corp ibm almaden research center campus in the santa teresa hills of san jose california photographer tony avelar bloomberg by your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least that s what international business machines corp sees in its crystal ball the predictions are part of an annual tradition for the armonk new york based company which surveys its researchers to find five ideas expected to take root in the next five years ibm the world s largest provider of computer services looks to silicon valley for input gleaning many ideas from its almaden research center in san jose california holographic conversations projected from mobile phones lead this year s list the predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers these are all stretch goals and that s good said paul saffo managing director of foresight at the investment advisory firm discern in san francisco in an era when pessimism is the new black a little dose of technological optimism is not a bad thing for ibm it s not just idle speculation the company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth saffo said not all of its predictions pan out though ibm was overly optimistic about the spread of speech technology for instance when the ideas do lead to products they can have broad implications for society as well as ibm s bottom line he said research spending they have continued to do research when all the other grand research organizations are gone said saffo who is also a consulting associate professor at stanford university ibm invested billion in research and development last year percent of revenue while that s down from about percent in the early s the company spends a bigger share on research than its computing rivals hewlett packard co the top maker of personal computers spent percent last year at almaden scientists work on projects that don t always fit in with ibm s computer business the lab s research includes efforts to develop an electric car battery that runs miles on one charge a filtration system for desalination and a program that shows changes in geographic data ibm rose cents to at a m in new york stock exchange composite trading the stock had gained percent this year before today citizen science the list is meant to give a window into the company s innovation engine said josephine cheng a vice president at ibm s almaden lab all this demonstrates a real culture of innovation at ibm and willingness to devote itself to solving some of the world s biggest problems she said many of the predictions are based on projects that ibm has in the works one of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative earlier this year ibm teamed up with the california state water resources control board and the city of san jose environmental services to help gather information about waterways researchers from almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions the hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves traffic predictors ibm also sees data helping shorten commutes in the next five years computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck batteries may last times longer in than today ibm says rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge the final prediction involves recycling the heat generated by computers and data centers almost half of the power used by data centers is currently spent keeping the computers cool ibm scientists say it would be better to harness that heat to warm houses and offices in ibm s first list of predictions compiled at the end of researchers said instantaneous speech translation would become the norm that hasn t happened yet while some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in star trek second life the company also predicted that online immersive environments such as second life would become more widespread while immersive video games are as popular as ever second life s growth has slowed internet users are flocking instead to the more d environments of facebook inc and twitter inc meanwhile a prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones the nice thing about the list is that it provokes thought saffo said if everything came true they wouldn t be doing their job to contact the reporter on this story ryan flinn in san francisco at rflinn bloomberg net to contact the editor responsible for this story tom giles at tgiles bloomberg net by your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least thatus what international business machines corp sees in its crystal ball bloomberg news ibm predicts holographic calls air breathing batteries by html |\n",
            "| the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races and that can be carried on a plane without the hassle too the omega e gun starting pistol omega it s easy to take for granted just how insanely close some olympic races are and how much the minutiae of it all can matter the perfect example is the traditional starting gun seems easy you pull a trigger and the race starts boom what people don t consider when a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others that s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise omega has been the watch company tasked as the official timekeeper of the olympic games since at the vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling about as far away as you can get probably while still clearly being a starting gun pull the trigger once and off the olympians go if it s pressed twice consecutively it signals a false start working through a speaker system is what eliminates any kind of advantage for athletes it s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them this year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time it wouldn t be an enormous difference omega timing board member peter hufcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete they all hear the start commands and signal at exactly the same moment there s also an ulterior reason for its look in a post september th world a gun on its way to a major event is going to raise more than a few tsa eyebrows even if it s a realistic looking fake rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun but there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some pavlovian response after more than a century of olympic games that make people want to hear the real thing not a whiny electronic noise everyone in the stands at home thankfully will still be getting that the sound is programmable and can be synthesized to sound like almost anything hufcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle too technologygadgetslondon london olympicsolympicsomegastarting gunssummer olympicstimingpopular sciencepopsci popsci technology article electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |\n",
            "| fruits that fight the flu fruits that fight the flu cold flu mens health apples the most popular source of antioxidants in our diet one apple has an antioxidant effect equivalent to mg of vitamin c apples are loaded with protective flavonoids which may prevent heart disease and cancer next papayas with percent of the rda of vitamin c a papaya can help kick a cold right out of your system the beta carotene and vitamins c and e in papayas reduce inflammation throughout the lessening the effects of asthma next cranberries cranberries have more antioxidants than other common fruits and veggies one serving has five times the amount in broccoli cranberries are a natural probiotic enhancing good bacteria levels in the gut and protecting it from foodborne illnesses next grapefruit loaded with vitamin c grapefruit also contains natural compounds called limonoids which can lower cholesterol the red varieties are a potent source of the cancer fighting substance lycopene next bananas one of the top food sources of vitamin b bananas help reduce fatigue depression stress and insomnia bananas are high in magnesium which keeps bones strong and potassium which helps prevent heart disease and high blood pressure next everything you need to know about cold and flu so you donut get sick this season at menus healthcom cold flu infection sore throat sneeze immunity germs allergies stay healthy sick contagious medicines cold medicine menshealth health flu fighting fruits cm mmc facebook mens health content health fight flu with fruit                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |\n",
            "| foolproof tips for better sleep there was a period in my life when i had a lot of problems with sleep it took me very long to fall asleep i was easily awaken and i simply wasn t getting enough of rest at night i didn t want to take medication and this led me to learn several tips and tricks that really helped me to overcome my insomnia some of these tips i try to follow regularly don t worry about not getting enough sleep try not to worry about how much you sleep such worrying can start a cycle of negative thoughts that contribute to a condition known as learned insomnia learned insomnia occurs when you worry so much about whether or not you will be able to get adequate sleep that the bedtime rituals and behavior actually trigger insomnia don t force yourself to sleep the very attempt of trying to do so actually awakes you making it more difficult to sleep go to bed only when you are feeling really tired and sleepy don t look at the alarm clock at night looking at the clock promotes increased anxiety and obsession about time body heating procedures some studies suggest that soaking in hot water before going to bed can ease the transition into a deeper sleep avoid oversleep don t oversleep to make up for a poor night s sleep doing so for even a couple of days can reset your clock and make it harder for you to sleep at night sex sex is a well known nighttime stress reliever healthy sex life enhances your relationship relaxes your releases happy chemicals and even promotes wellness and it welcomes sleep avoid alcohol as a sleeping aid avoid the use of alcohol in the late evening the most common myth found among people is that they believe alcohol helps in the sleep but the fact is alcohol may initially act as sedative but it produces a number of sleep impairing effects in the long run associate your bed and bedroom with sleep and sex only don t watch tv eat or read in bed although these things help some people sleep they can also give your brain the idea that bed isn t just for sleeping and this can keep you awake naps if you suffer from insomnia try not taking a nap if the goal is to sleep more during the night napping may steal hours desired later on if you re a regular napper and experiencing difficulty falling or staying asleep at night give up the nap and see what happens written by c simmons of healthassist net dumb little man shares ideas to make the everyday person more productive in life expect to read tips on finance saving money business and some diy for the house tipsdiymoneyfinanceadviceproductivityefficienttechnologysavingsoftwarebusinesstools dumblittleman foolproof tips for better sleep html                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |\n",
            "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Lower case everything\n",
        "data_nlp= data_nlp.withColumn(\"boilerplate\",lower(col('boilerplate')))\n",
        "data_nlp.select(\"boilerplate\").show(4,False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rrFnhLE8iD9e"
      },
      "source": [
        "# **NLP** 🦄"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D8RimRt8ilt8"
      },
      "source": [
        "Split text into words (Tokenizing)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2X8fiskQh0XZ",
        "outputId": "1686e84a-94be-4c8b-a1f5-ad93b8402de8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|urlid|boilerplate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |ac_Index|words                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |\n",
            "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "|4042 | ibm sees holographic calls air breathing batteries ibm sees holographic calls airbreathing batteries a sign stands outside the international business machines corp ibm almaden research center campus in san jose california photographer tony avelar bloomberg buildings stand at the international business machines corp ibm almaden research center campus in the santa teresa hills of san jose california photographer tony avelar bloomberg by your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least that s what international business machines corp sees in its crystal ball the predictions are part of an annual tradition for the armonk new york based company which surveys its researchers to find five ideas expected to take root in the next five years ibm the world s largest provider of computer services looks to silicon valley for input gleaning many ideas from its almaden research center in san jose california holographic conversations projected from mobile phones lead this year s list the predictions also include air breathing batteries computer programs that can tell when and where traffic jams will take place environmental information generated by sensors in cars and phones and cities powered by the heat thrown off by computer servers these are all stretch goals and that s good said paul saffo managing director of foresight at the investment advisory firm discern in san francisco in an era when pessimism is the new black a little dose of technological optimism is not a bad thing for ibm it s not just idle speculation the company is one of the few big corporations investing in long range research projects and it counts on innovation to fuel growth saffo said not all of its predictions pan out though ibm was overly optimistic about the spread of speech technology for instance when the ideas do lead to products they can have broad implications for society as well as ibm s bottom line he said research spending they have continued to do research when all the other grand research organizations are gone said saffo who is also a consulting associate professor at stanford university ibm invested billion in research and development last year percent of revenue while that s down from about percent in the early s the company spends a bigger share on research than its computing rivals hewlett packard co the top maker of personal computers spent percent last year at almaden scientists work on projects that don t always fit in with ibm s computer business the lab s research includes efforts to develop an electric car battery that runs miles on one charge a filtration system for desalination and a program that shows changes in geographic data ibm rose cents to at a m in new york stock exchange composite trading the stock had gained percent this year before today citizen science the list is meant to give a window into the company s innovation engine said josephine cheng a vice president at ibm s almaden lab all this demonstrates a real culture of innovation at ibm and willingness to devote itself to solving some of the world s biggest problems she said many of the predictions are based on projects that ibm has in the works one of this year s ideas that sensors in cars wallets and personal devices will give scientists better data about the environment is an expansion of the company s citizen science initiative earlier this year ibm teamed up with the california state water resources control board and the city of san jose environmental services to help gather information about waterways researchers from almaden created an application that lets smartphone users snap photos of streams and creeks and report back on conditions the hope is that these casual observations will help local and state officials who don t have the resources to do the work themselves traffic predictors ibm also sees data helping shorten commutes in the next five years computer programs will use algorithms and real time traffic information to predict which roads will have backups and how to avoid getting stuck batteries may last times longer in than today ibm says rather than using the current lithium ion technology new models could rely on energy dense metals that only need to interact with the air to recharge some electronic devices might ditch batteries altogether and use something similar to kinetic wristwatches which only need to be shaken to generate a charge the final prediction involves recycling the heat generated by computers and data centers almost half of the power used by data centers is currently spent keeping the computers cool ibm scientists say it would be better to harness that heat to warm houses and offices in ibm s first list of predictions compiled at the end of researchers said instantaneous speech translation would become the norm that hasn t happened yet while some programs can quickly translate electronic documents and instant messages and other apps can perform limited speech translation there s nothing widely available that acts like the universal translator in star trek second life the company also predicted that online immersive environments such as second life would become more widespread while immersive video games are as popular as ever second life s growth has slowed internet users are flocking instead to the more d environments of facebook inc and twitter inc meanwhile a prediction that mobile phones will act as a wallet ticket broker concierge bank and shopping assistant is coming true thanks to the explosion of smartphone applications consumers can pay bills through their banking apps buy movie tickets and get instant feedback on potential purchases all with a few taps on their phones the nice thing about the list is that it provokes thought saffo said if everything came true they wouldn t be doing their job to contact the reporter on this story ryan flinn in san francisco at rflinn bloomberg net to contact the editor responsible for this story tom giles at tgiles bloomberg net by your mobile phone will project a d image of anyone who calls and your laptop will be powered by kinetic energy at least thatus what international business machines corp sees in its crystal ball bloomberg news ibm predicts holographic calls air breathing batteries by html |3.0     |[ibm, sees, holographic, calls, air, breathing, batteries, ibm, sees, holographic, calls, airbreathing, batteries, a, sign, stands, outside, the, international, business, machines, corp, ibm, almaden, research, center, campus, in, san, jose, california, photographer, tony, avelar, bloomberg, buildings, stand, at, the, international, business, machines, corp, ibm, almaden, research, center, campus, in, the, santa, teresa, hills, of, san, jose, california, photographer, tony, avelar, bloomberg, by, your, mobile, phone, will, project, a, d, image, of, anyone, who, calls, and, your, laptop, will, be, powered, by, kinetic, energy, at, least, that, s, what, international, business, machines, corp, sees, in, its, crystal, ball, the, predictions, are, part, of, an, annual, tradition, for, the, armonk, new, york, based, company, which, surveys, its, researchers, to, find, five, ideas, expected, to, take, root, in, the, next, five, years, ibm, the, world, s, largest, provider, of, computer, services, looks, to, silicon, valley, for, input, gleaning, many, ideas, from, its, almaden, research, center, in, san, jose, california, holographic, conversations, projected, from, mobile, phones, lead, this, year, s, list, the, predictions, also, include, air, breathing, batteries, computer, programs, that, can, tell, when, and, where, traffic, jams, will, take, place, environmental, information, generated, by, sensors, in, cars, and, phones, and, cities, powered, by, the, heat, thrown, off, by, computer, servers, these, are, all, stretch, goals, and, that, s, good, said, paul, saffo, managing, director, of, foresight, at, the, investment, advisory, firm, discern, in, san, francisco, in, an, era, when, pessimism, is, the, new, black, a, little, dose, of, technological, optimism, is, not, a, bad, thing, for, ibm, it, s, not, just, idle, speculation, the, company, is, one, of, the, few, big, corporations, investing, in, long, range, research, projects, and, it, counts, on, innovation, to, fuel, growth, saffo, said, not, all, of, its, predictions, pan, out, though, ibm, was, overly, optimistic, about, the, spread, of, speech, technology, for, instance, when, the, ideas, do, lead, to, products, they, can, have, broad, implications, for, society, as, well, as, ibm, s, bottom, line, he, said, research, spending, they, have, continued, to, do, research, when, all, the, other, grand, research, organizations, are, gone, said, saffo, who, is, also, a, consulting, associate, professor, at, stanford, university, ibm, invested, billion, in, research, and, development, last, year, percent, of, revenue, while, that, s, down, from, about, percent, in, the, early, s, the, company, spends, a, bigger, share, on, research, than, its, computing, rivals, hewlett, packard, co, the, top, maker, of, personal, computers, spent, percent, last, year, at, almaden, scientists, work, on, projects, that, don, t, always, fit, in, with, ibm, s, computer, business, the, lab, s, research, includes, efforts, to, develop, an, electric, car, battery, that, runs, miles, on, one, charge, a, filtration, system, for, desalination, and, a, program, that, shows, changes, in, geographic, data, ibm, rose, cents, to, at, a, m, in, new, york, stock, exchange, composite, trading, the, stock, had, gained, percent, this, year, before, today, citizen, science, the, list, is, meant, to, give, a, window, into, the, company, s, innovation, engine, said, josephine, cheng, a, vice, president, at, ibm, s, almaden, lab, all, this, demonstrates, a, real, culture, of, innovation, at, ibm, and, willingness, to, devote, itself, to, solving, some, of, the, world, s, biggest, problems, she, said, many, of, the, predictions, are, based, on, projects, that, ibm, has, in, the, works, one, of, this, year, s, ideas, that, sensors, in, cars, wallets, and, personal, devices, will, give, scientists, better, data, about, the, environment, is, an, expansion, of, the, company, s, citizen, science, initiative, earlier, this, year, ibm, teamed, up, with, the, california, state, water, resources, control, board, and, the, city, of, san, jose, environmental, services, to, help, gather, information, about, waterways, researchers, from, almaden, created, an, application, that, lets, smartphone, users, snap, photos, of, streams, and, creeks, and, report, back, on, conditions, the, hope, is, that, these, casual, observations, will, help, local, and, state, officials, who, don, t, have, the, resources, to, do, the, work, themselves, traffic, predictors, ibm, also, sees, data, helping, shorten, commutes, in, the, next, five, years, computer, programs, will, use, algorithms, and, real, time, traffic, information, to, predict, which, roads, will, have, backups, and, how, to, avoid, getting, stuck, batteries, may, last, times, longer, in, than, today, ibm, says, rather, than, using, the, current, lithium, ion, technology, new, models, could, rely, on, energy, dense, metals, that, only, need, to, interact, with, the, air, to, recharge, some, electronic, devices, might, ditch, batteries, altogether, and, use, something, similar, to, kinetic, wristwatches, which, only, need, to, be, shaken, to, generate, a, charge, the, final, prediction, involves, recycling, the, heat, generated, by, computers, and, data, centers, almost, half, of, the, power, used, by, data, centers, is, currently, spent, keeping, the, computers, cool, ibm, scientists, say, it, would, be, better, to, harness, that, heat, to, warm, houses, and, offices, in, ibm, s, first, list, of, predictions, compiled, at, the, end, of, researchers, said, instantaneous, speech, translation, would, become, the, norm, that, hasn, t, happened, yet, while, some, programs, can, quickly, translate, electronic, documents, and, instant, messages, and, other, apps, can, perform, limited, speech, translation, there, s, nothing, widely, available, that, acts, like, the, universal, translator, in, star, trek, second, life, the, company, also, predicted, that, online, immersive, environments, such, as, second, life, would, become, more, widespread, while, immersive, video, games, are, as, popular, as, ever, second, life, s, growth, has, slowed, internet, users, are, flocking, instead, to, the, more, d, environments, of, facebook, inc, and, twitter, inc, meanwhile, a, prediction, that, mobile, phones, will, act, as, a, wallet, ticket, broker, concierge, bank, and, shopping, assistant, is, coming, true, thanks, to, the, explosion, of, smartphone, applications, consumers, can, pay, bills, through, their, banking, apps, buy, movie, tickets, and, get, instant, feedback, on, potential, purchases, all, with, a, few, taps, on, their, phones, the, nice, thing, about, the, list, is, that, it, provokes, thought, saffo, said, if, everything, came, true, they, wouldn, t, be, doing, their, job, to, contact, the, reporter, on, this, story, ryan, flinn, in, san, francisco, at, rflinn, bloomberg, net, to, contact, the, editor, responsible, for, this, story, tom, giles, at, tgiles, bloomberg, net, by, your, mobile, phone, will, project, a, d, image, of, anyone, who, calls, and, your, laptop, will, be, powered, by, kinetic, energy, at, least, thatus, what, international, business, machines, corp, sees, in, its, crystal, ball, bloomberg, news, ibm, predicts, holographic, calls, air, breathing, batteries, by, html]|\n",
            "|8471 | the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races the fully electronic futuristic starting gun that eliminates advantages in races and that can be carried on a plane without the hassle too the omega e gun starting pistol omega it s easy to take for granted just how insanely close some olympic races are and how much the minutiae of it all can matter the perfect example is the traditional starting gun seems easy you pull a trigger and the race starts boom what people don t consider when a conventional gun goes off the sound travels to the ears of the closest runner a fraction of a second sooner than the others that s just enough to matter and why the latest starting pistol has traded in the mechanical boom for orchestrated electronic noise omega has been the watch company tasked as the official timekeeper of the olympic games since at the vancouver games they debuted their new starting gun which is a far cry from the iconic revolvers associated with early games it s clearly electronic but still more than a button that s pressed to get the show rolling about as far away as you can get probably while still clearly being a starting gun pull the trigger once and off the olympians go if it s pressed twice consecutively it signals a false start working through a speaker system is what eliminates any kind of advantage for athletes it s not a big advantage being close to a gun but the sound of the bullet traveling one meter every three milliseconds could contribute to a win powder pistols have been connected to a speaker system before but even then runners could react to the sound of the real pistol firing rather than wait for the speaker sounds to reach them this year s setup will have speakers placed equidistant from runners forcing the sound to reach each competitor at exactly the same time it wouldn t be an enormous difference omega timing board member peter hufcrzeler said in an email but when you think about reaction times being measured in tiny fractions of a second placing a speaker behind each lane has eliminated any sort of advantage for any athlete they all hear the start commands and signal at exactly the same moment there s also an ulterior reason for its look in a post september th world a gun on its way to a major event is going to raise more than a few tsa eyebrows even if it s a realistic looking fake rather than deal with that the e gun can be transported while still maintaining the general look of a starting gun but there s still nothing like hearing a starting gun go off at the start of a race more than signaling the runners there s probably some pavlovian response after more than a century of olympic games that make people want to hear the real thing not a whiny electronic noise everyone in the stands at home thankfully will still be getting that the sound is programmable and can be synthesized to sound like almost anything hufcrzeler says but we program it to sound like a pistol it s a way to use the best possible starting technology but to keep a rich tradition alive and that can be carried on a plane without the hassle too technologygadgetslondon london olympicsolympicsomegastarting gunssummer olympicstimingpopular sciencepopsci popsci technology article electronic futuristic starting gun eliminates advantages races                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |1.0     |[the, fully, electronic, futuristic, starting, gun, that, eliminates, advantages, in, races, the, fully, electronic, futuristic, starting, gun, that, eliminates, advantages, in, races, the, fully, electronic, futuristic, starting, gun, that, eliminates, advantages, in, races, and, that, can, be, carried, on, a, plane, without, the, hassle, too, the, omega, e, gun, starting, pistol, omega, it, s, easy, to, take, for, granted, just, how, insanely, close, some, olympic, races, are, and, how, much, the, minutiae, of, it, all, can, matter, the, perfect, example, is, the, traditional, starting, gun, seems, easy, you, pull, a, trigger, and, the, race, starts, boom, what, people, don, t, consider, when, a, conventional, gun, goes, off, the, sound, travels, to, the, ears, of, the, closest, runner, a, fraction, of, a, second, sooner, than, the, others, that, s, just, enough, to, matter, and, why, the, latest, starting, pistol, has, traded, in, the, mechanical, boom, for, orchestrated, electronic, noise, omega, has, been, the, watch, company, tasked, as, the, official, timekeeper, of, the, olympic, games, since, at, the, vancouver, games, they, debuted, their, new, starting, gun, which, is, a, far, cry, from, the, iconic, revolvers, associated, with, early, games, it, s, clearly, electronic, but, still, more, than, a, button, that, s, pressed, to, get, the, show, rolling, about, as, far, away, as, you, can, get, probably, while, still, clearly, being, a, starting, gun, pull, the, trigger, once, and, off, the, olympians, go, if, it, s, pressed, twice, consecutively, it, signals, a, false, start, working, through, a, speaker, system, is, what, eliminates, any, kind, of, advantage, for, athletes, it, s, not, a, big, advantage, being, close, to, a, gun, but, the, sound, of, the, bullet, traveling, one, meter, every, three, milliseconds, could, contribute, to, a, win, powder, pistols, have, been, connected, to, a, speaker, system, before, but, even, then, runners, could, react, to, the, sound, of, the, real, pistol, firing, rather, than, wait, for, the, speaker, sounds, to, reach, them, this, year, s, setup, will, have, speakers, placed, equidistant, from, runners, forcing, the, sound, to, reach, each, competitor, at, exactly, the, same, time, it, wouldn, t, be, an, enormous, difference, omega, timing, board, member, peter, hufcrzeler, said, in, an, email, but, when, you, think, about, reaction, times, being, measured, in, tiny, fractions, of, a, second, placing, a, speaker, behind, each, lane, has, eliminated, any, sort, of, advantage, for, any, athlete, they, all, hear, the, start, commands, and, signal, at, exactly, the, same, moment, there, s, also, an, ulterior, reason, for, its, look, in, a, post, september, th, world, a, gun, on, its, way, to, a, major, event, is, going, to, raise, more, than, a, few, tsa, eyebrows, even, if, it, s, a, realistic, looking, fake, rather, than, deal, with, that, the, e, gun, can, be, transported, while, still, maintaining, the, general, look, of, a, starting, gun, but, there, s, still, nothing, like, hearing, a, starting, gun, go, off, at, the, start, of, a, race, more, than, signaling, the, runners, there, s, probably, some, pavlovian, response, after, more, than, a, century, of, olympic, games, that, make, people, want, to, hear, the, real, thing, not, a, whiny, electronic, noise, everyone, in, the, stands, at, home, thankfully, will, still, be, getting, that, the, sound, is, programmable, and, can, be, synthesized, to, sound, like, almost, anything, hufcrzeler, says, but, we, program, it, to, sound, like, a, pistol, it, s, a, way, to, use, the, best, possible, starting, technology, but, to, keep, a, rich, tradition, alive, and, that, can, be, carried, on, a, plane, without, the, hassle, too, technologygadgetslondon, london, olympicsolympicsomegastarting, gunssummer, olympicstimingpopular, sciencepopsci, popsci, technology, article, electronic, futuristic, starting, gun, eliminates, advantages, races]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |\n",
            "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
            "only showing top 2 rows\n",
            "\n",
            "root\n",
            " |-- urlid: string (nullable = true)\n",
            " |-- boilerplate: string (nullable = true)\n",
            " |-- ac_Index: double (nullable = false)\n",
            " |-- words: array (nullable = true)\n",
            " |    |-- element: string (containsNull = true)\n",
            "\n"
          ]
        }
      ],
      "source": [
        "regex_tokenizer = RegexTokenizer(inputCol=\"boilerplate\", outputCol=\"words\", pattern=\"\\\\W\")\n",
        "raw_words = regex_tokenizer.transform(data_nlp)\n",
        "raw_words.show(2,False)\n",
        "raw_words.printSchema()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BU70W6pHjCXx"
      },
      "source": [
        "Removing Stopwords"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y0LiP5SAi595",
        "outputId": "62ccf48f-77a8-4d98-e837-eacc4b3dce22"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your']"
            ]
          },
          "metadata": {},
          "execution_count": 184
        }
      ],
      "source": [
        "# from pyspark.ml.feature import StopWordsRemover\n",
        "\n",
        "# Define a list of stop words or use default list\n",
        "remover = StopWordsRemover(inputCol=\"words\", outputCol=\"filtered\")\n",
        "stopwords = remover.getStopWords() \n",
        "  \n",
        "# Display default list\n",
        "stopwords[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jh8akpRijSLM",
        "outputId": "8337bd48-ee03-42cf-9dea-240ee392011c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "|urlid|         boilerplate|ac_Index|               words|            filtered|\n",
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "| 4042| ibm sees hologra...|     3.0|[ibm, sees, holog...|[ibm, sees, holog...|\n",
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "only showing top 1 row\n",
            "\n"
          ]
        }
      ],
      "source": [
        "words_df = remover.transform(raw_words)\n",
        "words_df.show(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ywstfcLjg3z",
        "outputId": "b9531a75-1561-445e-d797-6d6807c8df3c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "|urlid|         boilerplate|ac_Index|               words|            filtered|\n",
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "| 4042| ibm sees hologra...|     3.0|[ibm, sees, holog...|[ibm, sees, holog...|\n",
            "+-----+--------------------+--------+--------------------+--------------------+\n",
            "only showing top 1 row\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Create the Pipeline\n",
        "pipeline = Pipeline(stages=[regex_tokenizer,remover])\n",
        "data_prep_pl = pipeline.fit(data_nlp)\n",
        "\n",
        "# Now call on the Pipeline to get our final df\n",
        "feature_data = data_prep_pl.transform(data_nlp)\n",
        "feature_data.show(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eXXE-vvWkJ5W"
      },
      "source": [
        "**Converting text into vectors**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4_9gTCLvkXfs"
      },
      "source": [
        "I will test out the **Word2Vec** vectorizor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YRR6mEis2dH8"
      },
      "outputs": [],
      "source": [
        "from pyspark.sql import Window\n",
        "from pyspark.sql.functions import row_number\n",
        "from pyspark.sql import functions as F"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PUXMC-G12pBh"
      },
      "outputs": [],
      "source": [
        "data_tp5=spark.createDataFrame(df_cleaned)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AeJpKN0i2nRf",
        "outputId": "98dfc41c-16ed-4d25-910f-acc65213a543"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-----+------+\n",
            "|            features|label|row_id|\n",
            "+--------------------+-----+------+\n",
            "|[0.789131,2.05555...|  0.0|     0|\n",
            "|[0.574147,3.67796...|  1.0|     1|\n",
            "|[0.996526,2.38288...|  1.0|     2|\n",
            "|[0.801248,1.54310...|  1.0|     3|\n",
            "|[0.719157,2.67647...|  0.0|     4|\n",
            "|[0.0,119.0,0.7454...|  0.0|     5|\n",
            "|[0.22111,0.773809...|  1.0|     6|\n",
            "|[0.0,1.883333333,...|  0.0|     7|\n",
            "|[0.0,0.471502591,...|  1.0|     8|\n",
            "|[0.0,2.41011236,0...|  1.0|     9|\n",
            "|[0.816604,2.50652...|  0.0|    10|\n",
            "|[0.89156,4.986111...|  0.0|    11|\n",
            "|[0.872323,3.05691...|  1.0|    12|\n",
            "|[0.0,0.0,0.0,0.0,...|  1.0|    13|\n",
            "|[0.559991,2.29949...|  0.0|    14|\n",
            "|[0.548963,0.99043...|  0.0|    15|\n",
            "|[0.598149,0.92982...|  0.0|    16|\n",
            "|[0.77292,2.083333...|  0.0|    17|\n",
            "|[0.287884,3.16438...|  1.0|    18|\n",
            "|[0.585389,2.82677...|  1.0|    19|\n",
            "+--------------------+-----+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "df = data_tp5.withColumn('row_id',F.monotonically_increasing_id())\n",
        "df.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4JFeep-RmUAu"
      },
      "outputs": [],
      "source": [
        "# Word2Vec\n",
        "word2Vec = Word2Vec(vectorSize=15, minCount=0, inputCol=\"filtered\", outputCol=\"features\")\n",
        "model = word2Vec.fit(feature_data)\n",
        "\n",
        "W2VfeaturizedData = model.transform(feature_data)\n",
        "\n",
        "scaler = MinMaxScaler(inputCol=\"features\", outputCol=\"scaledFeatures\")\n",
        "\n",
        "# Compute summary statistics and generate MinMaxScalerModel\n",
        "scalerModel = scaler.fit(W2VfeaturizedData)\n",
        "\n",
        "# rescale each feature to range [min, max].\n",
        "scaled_data = scalerModel.transform(W2VfeaturizedData)\n",
        "W2VfeaturizedData = scaled_data.select('urlid','ac_index','scaledFeatures')\n",
        "W2VfeaturizedData = W2VfeaturizedData.withColumnRenamed('scaledFeatures','features1')\n",
        "\n",
        "#joining both dataframes in order to apply classification models\n",
        "\n",
        "#adding row_id to data_nlp so that I can easily join it with dataframe containing features (5->22) and target\n",
        "df1 = W2VfeaturizedData.withColumn('row_id',F.monotonically_increasing_id())\n",
        "W2VfeaturizedData = df1.join(df, ['row_id'], 'left')\n",
        "#deleting row_id, ac_index and urlid after joining\n",
        "W2VfeaturizedData = W2VfeaturizedData.drop('row_id')\n",
        "#W2VfeaturizedData = W2VfeaturizedData.drop('ac_index')\n",
        "W2VfeaturizedData = W2VfeaturizedData.drop('urlid')\n",
        "W2VfeaturizedData = W2VfeaturizedData.drop('features')\n",
        "#W2VfeaturizedData =W2VfeaturizedData.withColumn(\"features\", concat(col(\"features1\"),col(\"features\")))\n",
        "W2VfeaturizedData = W2VfeaturizedData.withColumnRenamed('features1','features')\n",
        "\n",
        "W2VfeaturizedData.name = 'W2VfeaturizedData' # We will need this to print later"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XzYoqraKn1zx",
        "outputId": "5a2a51ab-81e7-4fb7-e931-2024f88e9f03"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------------------+-----+\n",
            "|ac_index|            features|label|\n",
            "+--------+--------------------+-----+\n",
            "|     3.0|[0.67583097061772...|  0.0|\n",
            "|     1.0|[0.58394053032566...|  1.0|\n",
            "+--------+--------------------+-----+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "W2VfeaturizedData.show(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-cRTEEZFpICL",
        "outputId": "ed907958-294a-4a7b-9df3-63534326fb40"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+\n",
            "|            features|\n",
            "+--------------------+\n",
            "|[0.65520995706264...|\n",
            "|[0.67583097061772...|\n",
            "|[0.56898178711883...|\n",
            "|[0.55267512339896...|\n",
            "|[0.52639653439018...|\n",
            "|[0.65943147184192...|\n",
            "|[0.68070165355285...|\n",
            "|[0.58394053032566...|\n",
            "|[0.65833521889253...|\n",
            "|[0.52076031896703...|\n",
            "|[0.55501059298462...|\n",
            "|[0.54475502905676...|\n",
            "|[0.53757659035358...|\n",
            "|[0.70983484955931...|\n",
            "|[0.57145139841587...|\n",
            "|[0.41304890792798...|\n",
            "|[0.53326572117392...|\n",
            "|[0.61406021407483...|\n",
            "|[0.55379945010500...|\n",
            "|[0.61677608500293...|\n",
            "+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "W2VfeaturizedData.select('features').show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DguwCC8FUgAc"
      },
      "source": [
        "**Train and evaluate our models**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2EYDIqnilmDV",
        "outputId": "da4b6b05-4315-4fc2-a3cf-b5156f6192a5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+--------------------+-----+\n",
            "|ac_index|            features|label|\n",
            "+--------+--------------------+-----+\n",
            "|     3.0|[0.67583097061772...|  0.0|\n",
            "+--------+--------------------+-----+\n",
            "only showing top 1 row\n",
            "\n"
          ]
        }
      ],
      "source": [
        "W2VfeaturizedData.show(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jer9uVSumY1-"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.linalg import Vector as MLVector, Vectors as MLVectors\n",
        "from pyspark.mllib.linalg import Vector as MLLibVector, Vectors as MLLibVectors\n",
        "from pyspark.mllib.regression import  LabeledPoint\n",
        "#labeled = W2VfeaturizedData.rdd.map(lambda row : LabeledPoint(row.get(\"label\"), row.get(\"features\")))\n",
        "labeled = W2VfeaturizedData.rdd.map(lambda row : LabeledPoint(int(row[2]),MLLibVectors.fromML(row[1]),))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b_PS2mGnOYUp"
      },
      "outputs": [],
      "source": [
        "(training , test) =labeled.randomSplit([0.8,0.2])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IoZr9YKHnx_b",
        "outputId": "bb96c10a-5d26-47f1-f846-26963b2f8f27"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabeledPoint(0.0, [0.6158325789764636,0.5929008474103011,0.6044689458569324,0.350053691883606,0.6458054635104243,0.5996900292465736,0.6390490194832239,0.3058508499687777,0.6522262589138477,0.6844004773424611,0.505477303125482,0.580975805911345,0.5101277942451384,0.4874318628078161,0.364511244369038])"
            ]
          },
          "metadata": {},
          "execution_count": 197
        }
      ],
      "source": [
        "\n",
        "labeled.first()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rs3yY5YylViP",
        "outputId": "52f4d823-655c-430f-844b-fd8b109aa266"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.8066666666666666\n"
          ]
        }
      ],
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.05)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebb24eef-1650-4817-eeb2-986c5ffbbddb",
        "id": "ItkqB1MjPmG5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.71\n"
          ]
        }
      ],
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.05)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###**essayons de regler les parametres des 2 modèles**"
      ],
      "metadata": {
        "id": "ZFblbnOoPmG6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "model 1 : LogisticRegressionWithSGD"
      ],
      "metadata": {
        "id": "pJP4Z45sPmG6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import LogisticRegressionWithSGD\n",
        "model = LogisticRegressionWithSGD.train(training, iterations = 1000 , regParam = 0.005)\n",
        "LabelPredict = test.map(lambda x : (x.label ,model.predict(x.features) ))\n",
        "trainingErr = LabelPredict.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"accuracy\", trainingErr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d1bf357d-a53b-48cd-a5a4-5b50f8952fa5",
        "id": "OsGmrAlNPmG6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pyspark/mllib/classification.py:327: FutureWarning: Deprecated in 2.0.0. Use ml.classification.LogisticRegression or LogisticRegressionWithLBFGS.\n",
            "  \"LogisticRegressionWithLBFGS.\", FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy 0.81\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "model 2: SVMWithSGD "
      ],
      "metadata": {
        "id": "F4KgBLBsPmG6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.mllib.classification import SVMWithSGD\n",
        "model2 = SVMWithSGD.train(training, iterations = 1000 , regParam = 0.005, miniBatchFraction=0.01)\n",
        "LabelPredict2 = test.map(lambda x : (x.label ,model2.predict(x.features) ))\n",
        "trainingErr2 = LabelPredict2.filter(lambda x : x[0] ==x[1]).count()/float(test.count())\n",
        "print(\"acc\", trainingErr2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80ee92e5-d92d-4f9e-ba85-e0d3f94028ea",
        "id": "0E-qddmmPmG6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc 0.8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "avec un regParam=0.005 et miniBatchFraction=0.01, la performance du modèle est devenue 80%! Which is good 👏"
      ],
      "metadata": {
        "id": "YbSiYHglPmG6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Conclusion 3** : After applying NLP , the accuracy has really improved for both models : it used to be 49% but it has became 81% 🙂"
      ],
      "metadata": {
        "id": "-DjApRDcQDHB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **General conclusion**:  NLP wins 🥇 with 81% , PCA got the second place 🥈 with 61%, then there is Pearson correlation which accuracy has not been improved !"
      ],
      "metadata": {
        "id": "E_A3tAT_QrmI"
      }
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}