03_scrape_news_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install newsapi-python\n",
    "\n",
    "# #Usage\n",
    "# from newsapi import NewsApiClient\n",
    "\n",
    "# # Init\n",
    "# newsapi = NewsApiClient(api_key='API_KEY')\n",
    "\n",
    "# # /v2/top-headlines\n",
    "# top_headlines = newsapi.get_top_headlines(q='bitcoin',\n",
    "#                                           sources='bbc-news,the-verge',\n",
    "#                                           category='business',\n",
    "#                                           language='en',\n",
    "#                                           country='us')\n",
    "\n",
    "# # /v2/everything\n",
    "# all_articles = newsapi.get_everything(q='bitcoin',\n",
    "#                                       sources='bbc-news,the-verge',\n",
    "#                                       domains='bbc.co.uk,techcrunch.com',\n",
    "#                                       from_param='2017-12-01',\n",
    "#                                       to='2017-12-12',\n",
    "#                                       language='en',\n",
    "#                                       sort_by='relevancy',\n",
    "#                                       page=2)\n",
    "\n",
    "# # /v2/sources\n",
    "# sources = newsapi.get_sources()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://newsapi.org/v2/top-headlines?country=us&apiKey=cded74bf8ca34ad5ae851465e5ed179f\n",
    "\n",
    "url = ('https://newsapi.org/v2/top-headlines?'\n",
    "       'country=us&'\n",
    "       'apiKey=cded74bf8ca34ad5ae851465e5ed179f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'ok', 'totalResults': 20, 'articles': [{'source': {'id': None, 'name': 'Cbssports.com'}, 'author': '', 'title': '2018 Ryder Cup results, scores: Europe beats back talented United States team to regain trophy', 'description': 'After residing in American possession for two years, the Ryder Cup trophy returns to the European side', 'url': 'https://www.cbssports.com/golf/news/2018-ryder-cup-results-scores-europe-beats-back-talented-united-states-team-to-regain-trophy/', 'urlToImage': 'https://sportshub.cbsistatic.com/i/r/2018/09/30/6b3d7960-22f8-48e1-9049-3c4747b7ca1b/thumbnail/770x433/40e61490ffe4ca5e8fa601cdca0aeacc/ian-poulter-celebrating-getty-ryder-cup.jpg', 'publishedAt': '2018-09-30T16:03:41Z', 'content': 'The 2018 Ryder Cup was dramatic for a short while during the Sunday singles matches, but in the end, Europe rolled to a clear 17.5-10.5 victory after entering the day leading 10-6. The United States team certainly made the hosts earn the victory with a downhi… [+4362 chars]'}, {'source': {'id': None, 'name': 'Cbssports.com'}, 'author': '', 'title': 'Josh Gordon officially active vs. Dolphins, ready to make his debut for Patriots', 'description': 'Gordon was acquired from the Browns two weeks ago but was inactive last week against the Lions', 'url': 'https://www.cbssports.com/nfl/news/theres-reportedly-a-good-chance-josh-gordon-makes-his-patriots-debut-vs-dolphins-on-sunday/', 'urlToImage': 'https://sportshub.cbsistatic.com/i/r/2018/09/20/b616c735-2328-4777-8864-8bf5a3ef7a01/thumbnail/770x433/d0647d4f58019b3e34fbcf59613736ae/josh-gordon.jpg', 'publishedAt': '2018-09-30T15:50:11Z', 'content': \"Just short of two weeks after he was acquired from the Cleveland Browns, Josh Gordon will make his season debut for the New England Patriots. The Patriots released their inactives list on Sunday morning, and Gordon is not on it. New England's offense is off t… [+2066 chars]\"}, {'source': {'id': None, 'name': 'Salon.com'}, 'author': 'Matthew Rozsa', 'title': 'Donald Trump on Kim Jong-un: “We fell in love”', 'description': 'Trump has a long history of praising foreign dictators and has failed to acquire concessions from North Korea', 'url': 'https://www.salon.com/2018/09/30/donald-trump-praises-kim-jong-uns-personal-letter-as-a-beautiful-piece-of-art/', 'urlToImage': 'https://media.salon.com/2017/12/kim-jong-un7.jpg', 'publishedAt': '2018-09-30T15:46:09Z', 'content': 'President Donald Trump offered fulsome praise for North Korean dictator Kim Jong-un after receiving a pair of letters which the American leader described as \"beautiful,\" even claiming that he and the notorious despot \"fell in love.\" \"Yes, I agree that the rhe… [+3105 chars]'}, {'source': {'id': 'the-washington-post', 'name': 'The Washington Post'}, 'author': None, 'title': \"Kellyanne Conway: 'I'm a victim of sexual assault'\", 'description': 'The searing personal revelation on CNN came during a discussion on Supreme Court nominee Brett Kavanaugh.', 'url': 'https://www.washingtonpost.com/politics/2018/09/30/kellyanne-conway-im-victim-sexual-assault/', 'urlToImage': 'https://www.washingtonpost.com/resizer/Rr0XmJIOWMquLwSwDj_4_svm0eo=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/4VUOUNV76UI6RH2PUG326JK2UU.jpg', 'publishedAt': '2018-09-30T15:27:17Z', 'content': 'Kellyanne Conway cleared her throat for what she was about to say. “I’m a victim of sexual assault,” President Trump’s adviser told CNN on Sunday. Conway and anchor Jake Tapper were discussing the political edges of sexual assault allegations against Brett M.… [+3146 chars]'}, {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'}, 'author': None, 'title': \"Kanye West Ends 'SNL' With Speech About Trump and Bullying\", 'description': 'The monologue, delivered in a red Make America Great Again hat, discussed racism and Democrats.', 'url': 'https://www.nytimes.com/2018/09/30/arts/music/kanye-west-donald-trump-snl.html', 'urlToImage': 'https://static01.nyt.com/images/2018/10/01/arts/music/01snlkanye3/01snlkanye3-facebookJumbo.jpg', 'publishedAt': '2018-09-30T15:05:40Z', 'content': 'Some conservatives, including President Trump himself, have praised Mr. West for his position. “Thank you Kanye for your support,” the president wrote on Twitter. “It is making a big difference!” “Ye,” one of a string of albums Mr. West had his hand in this s… [+968 chars]'}, {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'}, 'author': None, 'title': \"Indonesia Tsunami Toll Soars Above 800. 'It Is Very Bad.'\", 'description': 'The confirmed death toll has passed 800, officials said, but that is expected to rise significantly as rescuers scramble to reach the worst-hit areas.', 'url': 'https://www.nytimes.com/2018/09/30/world/asia/indonesia-tsunami.html', 'urlToImage': 'https://static01.nyt.com/images/2018/10/01/world/01tsunami-1/01tsunami-1-facebookJumbo.jpg', 'publishedAt': '2018-09-30T14:48:45Z', 'content': 'While World Vision’s staff from Donggala have made it safely to Palu city, where employees are sheltering in tarpaulin shelters set up in the courtyard of their office, they passed scenes of devastation on the way, Mr. Doseba said. “They told me they saw lots… [+1843 chars]'}, {'source': {'id': 'the-washington-post', 'name': 'The Washington Post'}, 'author': 'https://www.facebook.com/simon.denyer?fref=ts', 'title': 'Opponent of US military bases wins Okinawa gubernatorial election', 'description': 'Denny Tamaki wants a major reduction in the U.S. military presence on this strategically vital Japanese island.', 'url': 'https://www.washingtonpost.com/world/asia_pacific/opponent-of-us-military-bases-wins-okinawa-gubernatorial-election/2018/09/30/18f13d5a-c36a-11e8-9451-e878f96be19b_story.html', 'urlToImage': 'https://www.washingtonpost.com/resizer/qqWFzL4Q2pOBa4UcMBHp7lUzBtM=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/QBCKV4GCP4I6RFCR5B4PS27BTM.jpg', 'publishedAt': '2018-09-30T14:40:28Z', 'content': 'TOKYO — Denny Tamaki was elected Sunday as the next governor of Okinawa prefecture in Japan, after a campaign focused on sharply reducing the U.S. military presence on the island. Okinawa is home to about half the 54,000 U.S. troops stationed in Japan and hou… [+4732 chars]'}, {'source': {'id': 'fox-news', 'name': 'Fox News'}, 'author': 'Travis Fedschun', 'title': 'Allentown, Pennsylvania, vehicle explosion leaves at least 1 dead, officials say', 'description': \"At least one person was killed after a car exploded Saturday night in Pennsylvania's third-largest city, drawing a massive law enforcement response and involvement of federal officials.\", 'url': 'http://www.foxnews.com/us/2018/09/30/allentown-pennsylvania-vehicle-explosion-leaves-at-least-1-dead-officials-say.html', 'urlToImage': 'http://a57.foxnews.com/media2.foxnews.com/BrightCove/694940094001/2018/09/30/0/0/694940094001_5842386997001_5842385055001-vs.jpg?ve=1', 'publishedAt': '2018-09-30T14:23:03Z', 'content': \"At least one person was killed after a car exploded Saturday night in Pennsylvania's third-largest city, drawing a massive law enforcement response and involvement of federal officials. The Allentown Police Department said in a statement to Fox News the blast… [+2279 chars]\"}, {'source': {'id': 'the-guardian-au', 'name': 'The Guardian (AU)'}, 'author': 'Ed Pilkington', 'title': \"Willie Nelson lends power of his legend to Beto O'Rourke on Texas stage\", 'description': 'Nelson holds Austin concert in support of the Democratic candidate who is trying to unseat Ted Cruz from his Senate seat', 'url': 'https://www.theguardian.com/music/2018/sep/30/willie-nelson-concert-beto-orourke-texas', 'urlToImage': 'https://i.guim.co.uk/img/media/866cca5fb0a1a34f7582218896e51ef6f20603bd/0_359_4200_2521/master/4200.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&crop=faces%2Centropy&bm=normal&ba=bottom%2Cleft&blend64=aHR0cHM6Ly9hc3NldHMuZ3VpbS5jby51ay9pbWFnZXMvb3ZlcmxheXMvZDM1ODZhNWVmNTc4MTc1NmQyMWEzYjYzNWU1MTcxNDEvdGctZGVmYXVsdC5wbmc&s=89b297e1725c21426c075c22ee1d0022', 'publishedAt': '2018-09-30T13:23:00Z', 'content': 'Willie Nelson took to the stage in Austin on Saturday night wearing a “Beto for Texas” cap over his trademark braided hair. It didn’t last long – within minutes he had flung the hat into the crowd and replaced it with his trusted red bandana. That Nelson was … [+4472 chars]'}, {'source': {'id': 'nfl-news', 'name': 'NFL News'}, 'author': 'Austin Knoblauch', 'title': \"Pittsburgh Steelers actively shopping Le'Veon Bell\", 'description': 'The Pittsburgh Steelers are looking to part ways with running back Le\\'Veon Bell -- if another team is willing to trade away at least a second-round draft pick and a \"good\" player for him.', 'url': 'http://www.nfl.com/news/story/0ap3000000967510/article/pittsburgh-steelers-actively-shopping-leveon-bell', 'urlToImage': 'http://static.nfl.com/static/content/public/photo/2018/09/23/0ap3000000964920_thumbnail_200_150.jpg', 'publishedAt': '2018-09-30T12:37:00Z', 'content': \"The Pittsburgh Steelers are looking to part ways with running back Le'Veon Bell -- if another team is willing to pay a steep price. With just over a month to go before the NFL trade deadline, the Steelers are contacting teams and actively shopping Bell, a sou… [+1191 chars]\"}, {'source': {'id': 'newsweek', 'name': 'Newsweek'}, 'author': 'Tom Porter', 'title': \"Sarah Palin's Eldest Son Arrested on Domestic Violence Charges\", 'description': \"It is the third time the former Alaska governor's son has faced domestic violence charges.\", 'url': 'https://www.newsweek.com/sarah-palins-eldest-son-arrested-domestic-violence-charges-1145233', 'urlToImage': 'https://s.newsweek.com/sites/www.newsweek.com/files/styles/full/public/2018/09/30/gettyimages-82653473.jpg', 'publishedAt': '2018-09-30T12:31:37Z', 'content': 'Track Palin, the eldest son of former Alaska governor and vice presidential candidate Sarah Palin, has been arrested on assault charges. Palin, 29, of Wasilla, Alaska, was arrested on suspicion of domestic violence, interfering with a report of domestic viole… [+1620 chars]'}, {'source': {'id': None, 'name': 'Dailystar.co.uk'}, 'author': 'Charlotte Ikonen', 'title': 'Las Vegas mass shooting conspiracy theories revealed including THIS bombshell claim', 'description': 'ISIS, Illuminati and Antifa are just some of the bizarre theories...', 'url': 'https://www.dailystar.co.uk/news/world-news/732838/las-vegas-shooting-stephen-paddock-motive-conspiracy-mandalay-hotel-anniversary-news', 'urlToImage': 'https://cdn.images.dailystar.co.uk/dynamic/204/photos/93000/936x622/1462093.jpg', 'publishedAt': '2018-09-30T11:40:00Z', 'content': 'Gunman Stephen Paddock, a 64-year-old OAP from Mesquite, Nevada, fired hundreds of bullets at revellers enjoying the Route 91 Harvest Festival on October 2, 2017. He carried out the terrifying assault rifle rampage from the balcony of his 32nd floor Mandalay … [+9055 chars]'}, {'source': {'id': 'reuters', 'name': 'Reuters'}, 'author': 'Idrees Ali', 'title': 'US warship sails near disputed South China Sea islands: US official', 'description': 'A U.S. Navy destroyer sailed near islands claimed by China in the South China Sea on Sunday, a U.S. official told Reuters, potentially angering Beijing at a time of tense relations between the two countries.', 'url': 'https://www.reuters.com/article/us-usa-china/u-s-warship-sails-near-disputed-south-china-sea-islands-official-idUSKCN1MA0DY', 'urlToImage': 'https://s4.reutersmedia.net/resources/r/?m=02&d=20180930&t=2&i=1309732766&w=1200&r=LYNXNPEE8T0EU', 'publishedAt': '2018-09-30T11:09:11Z', 'content': 'WASHINGTON (Reuters) - A U.S. Navy destroyer sailed near islands claimed by China in the South China Sea on Sunday, a U.S. official told Reuters, potentially angering Beijing at a time of tense relations between the two countries. Beijing and Washington are l… [+2120 chars]'}, {'source': {'id': 'abc-news', 'name': 'ABC News'}, 'author': 'DANIEL MANZO', 'title': \"Hurricane Rosa's remnants likely to bring widespread heavy rain, flash flooding to southwest US\", 'description': 'Rosa will make landfall in Baja California on Monday.', 'url': 'https://abcnews.go.com/US/hurricane-rosas-remnants-bring-widespread-heavy-rain-flash/story?id=58183580', 'urlToImage': 'https://s.abcnews.com/images/US/rosa-sat-abc-mo-20180930_hpMain_16x9_992.jpg', 'publishedAt': '2018-09-30T10:19:16Z', 'content': 'As forecast, Hurricane Rosa is weakening as it moves over the cooler waters of the northern coast of Mexico. However, Rosa will bring flooding rains across northern Mexico and the southwest U.S. over the coming days. Interested in Hurricanes? Add Hurricanes a… [+3448 chars]'}, {'source': {'id': 'newsweek', 'name': 'Newsweek'}, 'author': 'David Brennan', 'title': 'Will Macedonia Change its Name? Voters Hope to End Greece Dispute and Win EU and NATO Membership', 'description': 'A new name would end the dispute with neighboring Greece that has kept Macedonia out of NATO and the European Union.', 'url': 'https://www.newsweek.com/will-macedonia-change-its-name-voters-hope-end-greece-dispute-and-win-eu-and-1143810', 'urlToImage': 'https://s.newsweek.com/sites/www.newsweek.com/files/styles/full/public/2018/09/28/rts21k6m.jpg', 'publishedAt': '2018-09-30T10:00:02Z', 'content': 'Macedonians will go to the polls Sunday for a referendum that could mark the beginning of the young nation’s acceptance into some of the most powerful international blocs in the world. Macedonia has been caught in a naming dispute with neighboring Greece sinc… [+4333 chars]'}, {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'}, 'author': None, 'title': \"Elon Musk Steps Down as Tesla's Chairman in Settlement With SEC\", 'description': 'After walking away from a deal, Tesla’s chief agreed to give up the chairmanship for three years and pay a $20 million fine.', 'url': 'https://www.nytimes.com/2018/09/29/business/tesla-musk-sec-settlement.html', 'urlToImage': 'https://static01.nyt.com/images/2018/09/30/business/30tesla/merlin_143915241_201c28c1-6a39-4109-a296-3515bcd86d2d-facebookJumbo.jpg', 'publishedAt': '2018-09-29T23:37:30Z', 'content': 'The whipsaw events of the past few days followed a series of calamitous events at Tesla, many of them self-inflicted wounds caused by Mr. Musk and his tweet about going private. The S.E.C. reacted quickly after the Aug. 7 tweet, which caused an immediate surg… [+1338 chars]'}, {'source': {'id': None, 'name': 'Voanews.com'}, 'author': 'Steve Herman', 'title': 'Trump Optimistic About New Kavanaugh Inquiry', 'description': \"President tells reporters he expects weeklong FBI investigation 'to turn out very well' for high court nominee\", 'url': 'https://www.voanews.com/a/fbi-reportedly-reaches-out-to-second-kavanaugh-accuser/4592847.html', 'urlToImage': 'https://gdb.voanews.com/093EA303-A2F2-4186-9227-676C13AD153C_w1200_r1_s.jpg', 'publishedAt': '2018-09-29T23:28:21Z', 'content': 'WHITE HOUSE — Amid a new investigation of his Supreme Court nominee, U.S. President Donald Trump maintained his support for Judge Brett Kavanaugh on Saturday, saying that \"hopefully, at the conclusion, everything will be fine.\" Trump, speaking to reporters on… [+4037 chars]'}, {'source': {'id': None, 'name': 'Express.co.uk'}, 'author': None, 'title': 'Shark attack horror: Boy, 13, mauled in California while diving for lobsters', 'description': None, 'url': 'https://www.express.co.uk/news/world/1024693/california-shark-attack-beacon-beach-boy-13-injured-lobsters-Chad-Hammel', 'urlToImage': None, 'publishedAt': '2018-09-29T21:58:01Z', 'content': None}, {'source': {'id': None, 'name': 'Bostonglobe.com'}, 'author': 'Victoria McGrane - Reporter', 'title': \"Warren says she will 'take a hard look at running for president'\", 'description': '“After November 6 I will take a hard look at running for president,” US Senator Elizabeth Warren said in response to an audience member asked her if she was considering a bid.', 'url': 'https://www.bostonglobe.com/metro/2018/09/29/elizabeth-warren-says-she-will-take-hard-look-running-for-president-after-midterms/fXnwr21zz2OTlBrgoPrgoJ/story.html', 'urlToImage': 'https://www.bostonglobe.com/rf/image_585w/Boston/2011-2020/2018/09/29/BostonGlobe.com/Metro/Images/Swensen_180929_30warren_0002.jpg', 'publishedAt': '2018-09-29T21:02:49Z', 'content': 'HOLYOKE — Senator Elizabeth Warren on Saturday made her most definitive indication to date that she is mulling a run for the White House in 2020, telling a town hall crowd that she will take a “hard look at running for president” after the Nov. 6 midterm elec… [+7266 chars]'}, {'source': {'id': None, 'name': 'Yahoo.com'}, 'author': None, 'title': 'Memphis civic leader, former Pinnacle Airlines CEO shot dead', 'description': None, 'url': 'https://www.yahoo.com/news/former-pinnacle-airlines-ceo-shot-042453217.html', 'urlToImage': None, 'publishedAt': '2018-09-28T18:39:00Z', 'content': None}]}\n"
     ]
    }
   ],
   "source": [
    "print( response.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "headlines = response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "help(headlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['status', 'totalResults', 'articles'])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(headlines['articles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'source': {'id': None, 'name': 'Cbssports.com'},\n",
       "  'author': '',\n",
       "  'title': '2018 Ryder Cup results, scores: Europe beats back talented United States team to regain trophy',\n",
       "  'description': 'After residing in American possession for two years, the Ryder Cup trophy returns to the European side',\n",
       "  'url': 'https://www.cbssports.com/golf/news/2018-ryder-cup-results-scores-europe-beats-back-talented-united-states-team-to-regain-trophy/',\n",
       "  'urlToImage': 'https://sportshub.cbsistatic.com/i/r/2018/09/30/6b3d7960-22f8-48e1-9049-3c4747b7ca1b/thumbnail/770x433/40e61490ffe4ca5e8fa601cdca0aeacc/ian-poulter-celebrating-getty-ryder-cup.jpg',\n",
       "  'publishedAt': '2018-09-30T16:03:41Z',\n",
       "  'content': 'The 2018 Ryder Cup was dramatic for a short while during the Sunday singles matches, but in the end, Europe rolled to a clear 17.5-10.5 victory after entering the day leading 10-6. The United States team certainly made the hosts earn the victory with a downhi… [+4362 chars]'},\n",
       " {'source': {'id': None, 'name': 'Cbssports.com'},\n",
       "  'author': '',\n",
       "  'title': 'Josh Gordon officially active vs. Dolphins, ready to make his debut for Patriots',\n",
       "  'description': 'Gordon was acquired from the Browns two weeks ago but was inactive last week against the Lions',\n",
       "  'url': 'https://www.cbssports.com/nfl/news/theres-reportedly-a-good-chance-josh-gordon-makes-his-patriots-debut-vs-dolphins-on-sunday/',\n",
       "  'urlToImage': 'https://sportshub.cbsistatic.com/i/r/2018/09/20/b616c735-2328-4777-8864-8bf5a3ef7a01/thumbnail/770x433/d0647d4f58019b3e34fbcf59613736ae/josh-gordon.jpg',\n",
       "  'publishedAt': '2018-09-30T15:50:11Z',\n",
       "  'content': \"Just short of two weeks after he was acquired from the Cleveland Browns, Josh Gordon will make his season debut for the New England Patriots. The Patriots released their inactives list on Sunday morning, and Gordon is not on it. New England's offense is off t… [+2066 chars]\"},\n",
       " {'source': {'id': None, 'name': 'Salon.com'},\n",
       "  'author': 'Matthew Rozsa',\n",
       "  'title': 'Donald Trump on Kim Jong-un: “We fell in love”',\n",
       "  'description': 'Trump has a long history of praising foreign dictators and has failed to acquire concessions from North Korea',\n",
       "  'url': 'https://www.salon.com/2018/09/30/donald-trump-praises-kim-jong-uns-personal-letter-as-a-beautiful-piece-of-art/',\n",
       "  'urlToImage': 'https://media.salon.com/2017/12/kim-jong-un7.jpg',\n",
       "  'publishedAt': '2018-09-30T15:46:09Z',\n",
       "  'content': 'President Donald Trump offered fulsome praise for North Korean dictator Kim Jong-un after receiving a pair of letters which the American leader described as \"beautiful,\" even claiming that he and the notorious despot \"fell in love.\" \"Yes, I agree that the rhe… [+3105 chars]'},\n",
       " {'source': {'id': 'the-washington-post', 'name': 'The Washington Post'},\n",
       "  'author': None,\n",
       "  'title': \"Kellyanne Conway: 'I'm a victim of sexual assault'\",\n",
       "  'description': 'The searing personal revelation on CNN came during a discussion on Supreme Court nominee Brett Kavanaugh.',\n",
       "  'url': 'https://www.washingtonpost.com/politics/2018/09/30/kellyanne-conway-im-victim-sexual-assault/',\n",
       "  'urlToImage': 'https://www.washingtonpost.com/resizer/Rr0XmJIOWMquLwSwDj_4_svm0eo=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/4VUOUNV76UI6RH2PUG326JK2UU.jpg',\n",
       "  'publishedAt': '2018-09-30T15:27:17Z',\n",
       "  'content': 'Kellyanne Conway cleared her throat for what she was about to say. “I’m a victim of sexual assault,” President Trump’s adviser told CNN on Sunday. Conway and anchor Jake Tapper were discussing the political edges of sexual assault allegations against Brett M.… [+3146 chars]'},\n",
       " {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'},\n",
       "  'author': None,\n",
       "  'title': \"Kanye West Ends 'SNL' With Speech About Trump and Bullying\",\n",
       "  'description': 'The monologue, delivered in a red Make America Great Again hat, discussed racism and Democrats.',\n",
       "  'url': 'https://www.nytimes.com/2018/09/30/arts/music/kanye-west-donald-trump-snl.html',\n",
       "  'urlToImage': 'https://static01.nyt.com/images/2018/10/01/arts/music/01snlkanye3/01snlkanye3-facebookJumbo.jpg',\n",
       "  'publishedAt': '2018-09-30T15:05:40Z',\n",
       "  'content': 'Some conservatives, including President Trump himself, have praised Mr. West for his position. “Thank you Kanye for your support,” the president wrote on Twitter. “It is making a big difference!” “Ye,” one of a string of albums Mr. West had his hand in this s… [+968 chars]'},\n",
       " {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'},\n",
       "  'author': None,\n",
       "  'title': \"Indonesia Tsunami Toll Soars Above 800. 'It Is Very Bad.'\",\n",
       "  'description': 'The confirmed death toll has passed 800, officials said, but that is expected to rise significantly as rescuers scramble to reach the worst-hit areas.',\n",
       "  'url': 'https://www.nytimes.com/2018/09/30/world/asia/indonesia-tsunami.html',\n",
       "  'urlToImage': 'https://static01.nyt.com/images/2018/10/01/world/01tsunami-1/01tsunami-1-facebookJumbo.jpg',\n",
       "  'publishedAt': '2018-09-30T14:48:45Z',\n",
       "  'content': 'While World Vision’s staff from Donggala have made it safely to Palu city, where employees are sheltering in tarpaulin shelters set up in the courtyard of their office, they passed scenes of devastation on the way, Mr. Doseba said. “They told me they saw lots… [+1843 chars]'},\n",
       " {'source': {'id': 'the-washington-post', 'name': 'The Washington Post'},\n",
       "  'author': 'https://www.facebook.com/simon.denyer?fref=ts',\n",
       "  'title': 'Opponent of US military bases wins Okinawa gubernatorial election',\n",
       "  'description': 'Denny Tamaki wants a major reduction in the U.S. military presence on this strategically vital Japanese island.',\n",
       "  'url': 'https://www.washingtonpost.com/world/asia_pacific/opponent-of-us-military-bases-wins-okinawa-gubernatorial-election/2018/09/30/18f13d5a-c36a-11e8-9451-e878f96be19b_story.html',\n",
       "  'urlToImage': 'https://www.washingtonpost.com/resizer/qqWFzL4Q2pOBa4UcMBHp7lUzBtM=/1484x0/arc-anglerfish-washpost-prod-washpost.s3.amazonaws.com/public/QBCKV4GCP4I6RFCR5B4PS27BTM.jpg',\n",
       "  'publishedAt': '2018-09-30T14:40:28Z',\n",
       "  'content': 'TOKYO — Denny Tamaki was elected Sunday as the next governor of Okinawa prefecture in Japan, after a campaign focused on sharply reducing the U.S. military presence on the island. Okinawa is home to about half the 54,000 U.S. troops stationed in Japan and hou… [+4732 chars]'},\n",
       " {'source': {'id': 'fox-news', 'name': 'Fox News'},\n",
       "  'author': 'Travis Fedschun',\n",
       "  'title': 'Allentown, Pennsylvania, vehicle explosion leaves at least 1 dead, officials say',\n",
       "  'description': \"At least one person was killed after a car exploded Saturday night in Pennsylvania's third-largest city, drawing a massive law enforcement response and involvement of federal officials.\",\n",
       "  'url': 'http://www.foxnews.com/us/2018/09/30/allentown-pennsylvania-vehicle-explosion-leaves-at-least-1-dead-officials-say.html',\n",
       "  'urlToImage': 'http://a57.foxnews.com/media2.foxnews.com/BrightCove/694940094001/2018/09/30/0/0/694940094001_5842386997001_5842385055001-vs.jpg?ve=1',\n",
       "  'publishedAt': '2018-09-30T14:23:03Z',\n",
       "  'content': \"At least one person was killed after a car exploded Saturday night in Pennsylvania's third-largest city, drawing a massive law enforcement response and involvement of federal officials. The Allentown Police Department said in a statement to Fox News the blast… [+2279 chars]\"},\n",
       " {'source': {'id': 'the-guardian-au', 'name': 'The Guardian (AU)'},\n",
       "  'author': 'Ed Pilkington',\n",
       "  'title': \"Willie Nelson lends power of his legend to Beto O'Rourke on Texas stage\",\n",
       "  'description': 'Nelson holds Austin concert in support of the Democratic candidate who is trying to unseat Ted Cruz from his Senate seat',\n",
       "  'url': 'https://www.theguardian.com/music/2018/sep/30/willie-nelson-concert-beto-orourke-texas',\n",
       "  'urlToImage': 'https://i.guim.co.uk/img/media/866cca5fb0a1a34f7582218896e51ef6f20603bd/0_359_4200_2521/master/4200.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&crop=faces%2Centropy&bm=normal&ba=bottom%2Cleft&blend64=aHR0cHM6Ly9hc3NldHMuZ3VpbS5jby51ay9pbWFnZXMvb3ZlcmxheXMvZDM1ODZhNWVmNTc4MTc1NmQyMWEzYjYzNWU1MTcxNDEvdGctZGVmYXVsdC5wbmc&s=89b297e1725c21426c075c22ee1d0022',\n",
       "  'publishedAt': '2018-09-30T13:23:00Z',\n",
       "  'content': 'Willie Nelson took to the stage in Austin on Saturday night wearing a “Beto for Texas” cap over his trademark braided hair. It didn’t last long – within minutes he had flung the hat into the crowd and replaced it with his trusted red bandana. That Nelson was … [+4472 chars]'},\n",
       " {'source': {'id': 'nfl-news', 'name': 'NFL News'},\n",
       "  'author': 'Austin Knoblauch',\n",
       "  'title': \"Pittsburgh Steelers actively shopping Le'Veon Bell\",\n",
       "  'description': 'The Pittsburgh Steelers are looking to part ways with running back Le\\'Veon Bell -- if another team is willing to trade away at least a second-round draft pick and a \"good\" player for him.',\n",
       "  'url': 'http://www.nfl.com/news/story/0ap3000000967510/article/pittsburgh-steelers-actively-shopping-leveon-bell',\n",
       "  'urlToImage': 'http://static.nfl.com/static/content/public/photo/2018/09/23/0ap3000000964920_thumbnail_200_150.jpg',\n",
       "  'publishedAt': '2018-09-30T12:37:00Z',\n",
       "  'content': \"The Pittsburgh Steelers are looking to part ways with running back Le'Veon Bell -- if another team is willing to pay a steep price. With just over a month to go before the NFL trade deadline, the Steelers are contacting teams and actively shopping Bell, a sou… [+1191 chars]\"},\n",
       " {'source': {'id': 'newsweek', 'name': 'Newsweek'},\n",
       "  'author': 'Tom Porter',\n",
       "  'title': \"Sarah Palin's Eldest Son Arrested on Domestic Violence Charges\",\n",
       "  'description': \"It is the third time the former Alaska governor's son has faced domestic violence charges.\",\n",
       "  'url': 'https://www.newsweek.com/sarah-palins-eldest-son-arrested-domestic-violence-charges-1145233',\n",
       "  'urlToImage': 'https://s.newsweek.com/sites/www.newsweek.com/files/styles/full/public/2018/09/30/gettyimages-82653473.jpg',\n",
       "  'publishedAt': '2018-09-30T12:31:37Z',\n",
       "  'content': 'Track Palin, the eldest son of former Alaska governor and vice presidential candidate Sarah Palin, has been arrested on assault charges. Palin, 29, of Wasilla, Alaska, was arrested on suspicion of domestic violence, interfering with a report of domestic viole… [+1620 chars]'},\n",
       " {'source': {'id': None, 'name': 'Dailystar.co.uk'},\n",
       "  'author': 'Charlotte Ikonen',\n",
       "  'title': 'Las Vegas mass shooting conspiracy theories revealed including THIS bombshell claim',\n",
       "  'description': 'ISIS, Illuminati and Antifa are just some of the bizarre theories...',\n",
       "  'url': 'https://www.dailystar.co.uk/news/world-news/732838/las-vegas-shooting-stephen-paddock-motive-conspiracy-mandalay-hotel-anniversary-news',\n",
       "  'urlToImage': 'https://cdn.images.dailystar.co.uk/dynamic/204/photos/93000/936x622/1462093.jpg',\n",
       "  'publishedAt': '2018-09-30T11:40:00Z',\n",
       "  'content': 'Gunman Stephen Paddock, a 64-year-old OAP from Mesquite, Nevada, fired hundreds of bullets at revellers enjoying the Route 91 Harvest Festival on October 2, 2017. He carried out the terrifying assault rifle rampage from the balcony of his 32nd floor Mandalay … [+9055 chars]'},\n",
       " {'source': {'id': 'reuters', 'name': 'Reuters'},\n",
       "  'author': 'Idrees Ali',\n",
       "  'title': 'US warship sails near disputed South China Sea islands: US official',\n",
       "  'description': 'A U.S. Navy destroyer sailed near islands claimed by China in the South China Sea on Sunday, a U.S. official told Reuters, potentially angering Beijing at a time of tense relations between the two countries.',\n",
       "  'url': 'https://www.reuters.com/article/us-usa-china/u-s-warship-sails-near-disputed-south-china-sea-islands-official-idUSKCN1MA0DY',\n",
       "  'urlToImage': 'https://s4.reutersmedia.net/resources/r/?m=02&d=20180930&t=2&i=1309732766&w=1200&r=LYNXNPEE8T0EU',\n",
       "  'publishedAt': '2018-09-30T11:09:11Z',\n",
       "  'content': 'WASHINGTON (Reuters) - A U.S. Navy destroyer sailed near islands claimed by China in the South China Sea on Sunday, a U.S. official told Reuters, potentially angering Beijing at a time of tense relations between the two countries. Beijing and Washington are l… [+2120 chars]'},\n",
       " {'source': {'id': 'abc-news', 'name': 'ABC News'},\n",
       "  'author': 'DANIEL MANZO',\n",
       "  'title': \"Hurricane Rosa's remnants likely to bring widespread heavy rain, flash flooding to southwest US\",\n",
       "  'description': 'Rosa will make landfall in Baja California on Monday.',\n",
       "  'url': 'https://abcnews.go.com/US/hurricane-rosas-remnants-bring-widespread-heavy-rain-flash/story?id=58183580',\n",
       "  'urlToImage': 'https://s.abcnews.com/images/US/rosa-sat-abc-mo-20180930_hpMain_16x9_992.jpg',\n",
       "  'publishedAt': '2018-09-30T10:19:16Z',\n",
       "  'content': 'As forecast, Hurricane Rosa is weakening as it moves over the cooler waters of the northern coast of Mexico. However, Rosa will bring flooding rains across northern Mexico and the southwest U.S. over the coming days. Interested in Hurricanes? Add Hurricanes a… [+3448 chars]'},\n",
       " {'source': {'id': 'newsweek', 'name': 'Newsweek'},\n",
       "  'author': 'David Brennan',\n",
       "  'title': 'Will Macedonia Change its Name? Voters Hope to End Greece Dispute and Win EU and NATO Membership',\n",
       "  'description': 'A new name would end the dispute with neighboring Greece that has kept Macedonia out of NATO and the European Union.',\n",
       "  'url': 'https://www.newsweek.com/will-macedonia-change-its-name-voters-hope-end-greece-dispute-and-win-eu-and-1143810',\n",
       "  'urlToImage': 'https://s.newsweek.com/sites/www.newsweek.com/files/styles/full/public/2018/09/28/rts21k6m.jpg',\n",
       "  'publishedAt': '2018-09-30T10:00:02Z',\n",
       "  'content': 'Macedonians will go to the polls Sunday for a referendum that could mark the beginning of the young nation’s acceptance into some of the most powerful international blocs in the world. Macedonia has been caught in a naming dispute with neighboring Greece sinc… [+4333 chars]'},\n",
       " {'source': {'id': 'the-new-york-times', 'name': 'The New York Times'},\n",
       "  'author': None,\n",
       "  'title': \"Elon Musk Steps Down as Tesla's Chairman in Settlement With SEC\",\n",
       "  'description': 'After walking away from a deal, Tesla’s chief agreed to give up the chairmanship for three years and pay a $20 million fine.',\n",
       "  'url': 'https://www.nytimes.com/2018/09/29/business/tesla-musk-sec-settlement.html',\n",
       "  'urlToImage': 'https://static01.nyt.com/images/2018/09/30/business/30tesla/merlin_143915241_201c28c1-6a39-4109-a296-3515bcd86d2d-facebookJumbo.jpg',\n",
       "  'publishedAt': '2018-09-29T23:37:30Z',\n",
       "  'content': 'The whipsaw events of the past few days followed a series of calamitous events at Tesla, many of them self-inflicted wounds caused by Mr. Musk and his tweet about going private. The S.E.C. reacted quickly after the Aug. 7 tweet, which caused an immediate surg… [+1338 chars]'},\n",
       " {'source': {'id': None, 'name': 'Voanews.com'},\n",
       "  'author': 'Steve Herman',\n",
       "  'title': 'Trump Optimistic About New Kavanaugh Inquiry',\n",
       "  'description': \"President tells reporters he expects weeklong FBI investigation 'to turn out very well' for high court nominee\",\n",
       "  'url': 'https://www.voanews.com/a/fbi-reportedly-reaches-out-to-second-kavanaugh-accuser/4592847.html',\n",
       "  'urlToImage': 'https://gdb.voanews.com/093EA303-A2F2-4186-9227-676C13AD153C_w1200_r1_s.jpg',\n",
       "  'publishedAt': '2018-09-29T23:28:21Z',\n",
       "  'content': 'WHITE HOUSE — Amid a new investigation of his Supreme Court nominee, U.S. President Donald Trump maintained his support for Judge Brett Kavanaugh on Saturday, saying that \"hopefully, at the conclusion, everything will be fine.\" Trump, speaking to reporters on… [+4037 chars]'},\n",
       " {'source': {'id': None, 'name': 'Express.co.uk'},\n",
       "  'author': None,\n",
       "  'title': 'Shark attack horror: Boy, 13, mauled in California while diving for lobsters',\n",
       "  'description': None,\n",
       "  'url': 'https://www.express.co.uk/news/world/1024693/california-shark-attack-beacon-beach-boy-13-injured-lobsters-Chad-Hammel',\n",
       "  'urlToImage': None,\n",
       "  'publishedAt': '2018-09-29T21:58:01Z',\n",
       "  'content': None},\n",
       " {'source': {'id': None, 'name': 'Bostonglobe.com'},\n",
       "  'author': 'Victoria McGrane - Reporter',\n",
       "  'title': \"Warren says she will 'take a hard look at running for president'\",\n",
       "  'description': '“After November 6 I will take a hard look at running for president,” US Senator Elizabeth Warren said in response to an audience member asked her if she was considering a bid.',\n",
       "  'url': 'https://www.bostonglobe.com/metro/2018/09/29/elizabeth-warren-says-she-will-take-hard-look-running-for-president-after-midterms/fXnwr21zz2OTlBrgoPrgoJ/story.html',\n",
       "  'urlToImage': 'https://www.bostonglobe.com/rf/image_585w/Boston/2011-2020/2018/09/29/BostonGlobe.com/Metro/Images/Swensen_180929_30warren_0002.jpg',\n",
       "  'publishedAt': '2018-09-29T21:02:49Z',\n",
       "  'content': 'HOLYOKE — Senator Elizabeth Warren on Saturday made her most definitive indication to date that she is mulling a run for the White House in 2020, telling a town hall crowd that she will take a “hard look at running for president” after the Nov. 6 midterm elec… [+7266 chars]'},\n",
       " {'source': {'id': None, 'name': 'Yahoo.com'},\n",
       "  'author': None,\n",
       "  'title': 'Memphis civic leader, former Pinnacle Airlines CEO shot dead',\n",
       "  'description': None,\n",
       "  'url': 'https://www.yahoo.com/news/former-pinnacle-airlines-ceo-shot-042453217.html',\n",
       "  'urlToImage': None,\n",
       "  'publishedAt': '2018-09-28T18:39:00Z',\n",
       "  'content': None}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines['articles']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': {'id': 'newsweek', 'name': 'Newsweek'},\n",
       " 'author': 'Tom Porter',\n",
       " 'title': \"Sarah Palin's Eldest Son Arrested on Domestic Violence Charges\",\n",
       " 'description': \"It is the third time the former Alaska governor's son has faced domestic violence charges.\",\n",
       " 'url': 'https://www.newsweek.com/sarah-palins-eldest-son-arrested-domestic-violence-charges-1145233',\n",
       " 'urlToImage': 'https://s.newsweek.com/sites/www.newsweek.com/files/styles/full/public/2018/09/30/gettyimages-82653473.jpg',\n",
       " 'publishedAt': '2018-09-30T12:31:37Z',\n",
       " 'content': 'Track Palin, the eldest son of former Alaska governor and vice presidential candidate Sarah Palin, has been arrested on assault charges. Palin, 29, of Wasilla, Alaska, was arrested on suspicion of domestic violence, interfering with a report of domestic viole… [+1620 chars]'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines['articles'][10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Josh Gordon officially active vs. Dolphins, ready to make his debut for Patriots'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines['articles'][1]['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "step_1 = headlines['articles'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2018 Ryder Cup results, scores: Europe beats back talented United States team to regain trophy'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "step_1['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['source', 'author', 'title', 'description', 'url', 'urlToImage', 'publishedAt', 'content'])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "step_1.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'keys'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-9079c9aeb22e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mheadlines\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'articles'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'keys'"
     ]
    }
   ],
   "source": [
    "headlines['articles'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "practice_pull = pd.DataFrame(headlines['articles'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author</th>\n",
       "      <th>content</th>\n",
       "      <th>description</th>\n",
       "      <th>publishedAt</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>urlToImage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2018-09-29T21:58:01Z</td>\n",
       "      <td>{'id': None, 'name': 'Express.co.uk'}</td>\n",
       "      <td>Shark attack horror: Boy, 13, mauled in Califo...</td>\n",
       "      <td>https://www.express.co.uk/news/world/1024693/c...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Charlotte Ikonen</td>\n",
       "      <td>Gunman Stephen Paddock, a 64-year-old OAP from...</td>\n",
       "      <td>ISIS, Illuminati and Antifa are just some of t...</td>\n",
       "      <td>2018-09-30T11:40:00Z</td>\n",
       "      <td>{'id': None, 'name': 'Dailystar.co.uk'}</td>\n",
       "      <td>Las Vegas mass shooting conspiracy theories re...</td>\n",
       "      <td>https://www.dailystar.co.uk/news/world-news/73...</td>\n",
       "      <td>https://cdn.images.dailystar.co.uk/dynamic/204...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Matthew Rozsa</td>\n",
       "      <td>President Donald Trump offered fulsome praise ...</td>\n",
       "      <td>Trump has a long history of praising foreign d...</td>\n",
       "      <td>2018-09-30T15:46:09Z</td>\n",
       "      <td>{'id': None, 'name': 'Salon.com'}</td>\n",
       "      <td>Donald Trump on Kim Jong-un: “We fell in love”</td>\n",
       "      <td>https://www.salon.com/2018/09/30/donald-trump-...</td>\n",
       "      <td>https://media.salon.com/2017/12/kim-jong-un7.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Victoria McGrane - Reporter</td>\n",
       "      <td>HOLYOKE — Senator Elizabeth Warren on Saturday...</td>\n",
       "      <td>“After November 6 I will take a hard look at r...</td>\n",
       "      <td>2018-09-29T21:02:49Z</td>\n",
       "      <td>{'id': None, 'name': 'Bostonglobe.com'}</td>\n",
       "      <td>Warren says she will 'take a hard look at runn...</td>\n",
       "      <td>https://www.bostonglobe.com/metro/2018/09/29/e...</td>\n",
       "      <td>https://www.bostonglobe.com/rf/image_585w/Bost...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Austin Knoblauch</td>\n",
       "      <td>The Pittsburgh Steelers are looking to part wa...</td>\n",
       "      <td>The Pittsburgh Steelers are looking to part wa...</td>\n",
       "      <td>2018-09-30T12:37:00Z</td>\n",
       "      <td>{'id': 'nfl-news', 'name': 'NFL News'}</td>\n",
       "      <td>Pittsburgh Steelers actively shopping Le'Veon ...</td>\n",
       "      <td>http://www.nfl.com/news/story/0ap3000000967510...</td>\n",
       "      <td>http://static.nfl.com/static/content/public/ph...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         author  \\\n",
       "17                         None   \n",
       "11             Charlotte Ikonen   \n",
       "2                 Matthew Rozsa   \n",
       "18  Victoria McGrane - Reporter   \n",
       "9              Austin Knoblauch   \n",
       "\n",
       "                                              content  \\\n",
       "17                                               None   \n",
       "11  Gunman Stephen Paddock, a 64-year-old OAP from...   \n",
       "2   President Donald Trump offered fulsome praise ...   \n",
       "18  HOLYOKE — Senator Elizabeth Warren on Saturday...   \n",
       "9   The Pittsburgh Steelers are looking to part wa...   \n",
       "\n",
       "                                          description           publishedAt  \\\n",
       "17                                               None  2018-09-29T21:58:01Z   \n",
       "11  ISIS, Illuminati and Antifa are just some of t...  2018-09-30T11:40:00Z   \n",
       "2   Trump has a long history of praising foreign d...  2018-09-30T15:46:09Z   \n",
       "18  “After November 6 I will take a hard look at r...  2018-09-29T21:02:49Z   \n",
       "9   The Pittsburgh Steelers are looking to part wa...  2018-09-30T12:37:00Z   \n",
       "\n",
       "                                     source  \\\n",
       "17    {'id': None, 'name': 'Express.co.uk'}   \n",
       "11  {'id': None, 'name': 'Dailystar.co.uk'}   \n",
       "2         {'id': None, 'name': 'Salon.com'}   \n",
       "18  {'id': None, 'name': 'Bostonglobe.com'}   \n",
       "9    {'id': 'nfl-news', 'name': 'NFL News'}   \n",
       "\n",
       "                                                title  \\\n",
       "17  Shark attack horror: Boy, 13, mauled in Califo...   \n",
       "11  Las Vegas mass shooting conspiracy theories re...   \n",
       "2      Donald Trump on Kim Jong-un: “We fell in love”   \n",
       "18  Warren says she will 'take a hard look at runn...   \n",
       "9   Pittsburgh Steelers actively shopping Le'Veon ...   \n",
       "\n",
       "                                                  url  \\\n",
       "17  https://www.express.co.uk/news/world/1024693/c...   \n",
       "11  https://www.dailystar.co.uk/news/world-news/73...   \n",
       "2   https://www.salon.com/2018/09/30/donald-trump-...   \n",
       "18  https://www.bostonglobe.com/metro/2018/09/29/e...   \n",
       "9   http://www.nfl.com/news/story/0ap3000000967510...   \n",
       "\n",
       "                                           urlToImage  \n",
       "17                                               None  \n",
       "11  https://cdn.images.dailystar.co.uk/dynamic/204...  \n",
       "2    https://media.salon.com/2017/12/kim-jong-un7.jpg  \n",
       "18  https://www.bostonglobe.com/rf/image_585w/Bost...  \n",
       "9   http://static.nfl.com/static/content/public/ph...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "practice_pull.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def query_pushshift(subreddit, kind='submission', skip=2, times=150, \n",
    "#                     subfield = ['title', 'selftext', 'subreddit', 'created_utc', 'author', 'num_comments', 'score', 'is_self']):\n",
    "\n",
    "#     stem = \"https://api.pushshift.io/reddit/search/{}/?subreddit={}&size=500\".format(kind, subreddit)\n",
    "#     mylist = []\n",
    "    \n",
    "#     for x in range(1, times):\n",
    "        \n",
    "#         URL = \"{}&after={}d\".format(stem, skip * x)\n",
    "#         print(URL)\n",
    "#         response = requests.get(URL)\n",
    "#         assert response.status_code == 200\n",
    "#         mine = response.json()['data']\n",
    "#         df = pd.DataFrame.from_dict(mine)\n",
    "#         mylist.append(df)\n",
    "#         time.sleep(0.25)\n",
    "        \n",
    "#     full = pd.concat(mylist)\n",
    "    \n",
    "#     if kind == \"submission\":\n",
    "        \n",
    "#         full = full[subfield]\n",
    "        \n",
    "#         full = full.drop_duplicates()\n",
    "        \n",
    "#         full = full.loc[full['is_self'] == True]\n",
    "        \n",
    "#     def get_date(created):\n",
    "#         return dt.date.fromtimestamp(created)\n",
    "    \n",
    "#     _timestamp = full[\"created_utc\"].apply(get_date)\n",
    "    \n",
    "#     full['timestamp'] = _timestamp\n",
    "\n",
    "#     print(full.shape)\n",
    "    \n",
    "#     return full "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "weather_url = ('https://newsapi.org/v2/top-headlines?'\n",
    "       'country=us&'\n",
    "       'q=weather&'\n",
    "       'apiKey=cded74bf8ca34ad5ae851465e5ed179f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "response_weather = requests.get(weather_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'ok', 'totalResults': 0, 'articles': []}\n"
     ]
    }
   ],
   "source": [
    "print(response_weather.json()) #time4:51pm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "weather_url = ('https://newsapi.org/v2/top-headlines?'\n",
    "       'country=us&'\n",
    "       'q=weather&'\n",
    "       'apiKey=cded74bf8ca34ad5ae851465e5ed179f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'ok', 'totalResults': 0, 'articles': []}\n"
     ]
    }
   ],
   "source": [
    "response_weather = requests.get(weather_url)\n",
    "print(response_weather.json())        #time 4:57pm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}