examples/intake-cmr-stac.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intake-STAC + NASA CMR\n",
    "\n",
    "NASA's Common Metadata Repository (CMR) now has a STAC endpoint https://github.com/nasa/cmr-stac. In theory we can use Intake-STAC to browse NASA's entire data catalog! \n",
    "\n",
    "**This example is experimental as the cmr-stac service is very new.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import intake\n",
    "import os\n",
    "import hvplot.xarray\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# testing remote reading of netcdf data\n",
    "import fsspec\n",
    "import aiohttp\n",
    "import netrc\n",
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search not currently working:\n",
    "# https://github.com/sat-utils/sat-search/issues/106\n",
    "#import satsearch\n",
    "#URL = 'https://cmr.earthdata.nasa.gov/cmr-stac/ASF'\n",
    "#results = satsearch.Search.search(url=URL,\n",
    "#                                  collections=['C1595422627-ASF']\n",
    "#                                  )\n",
    "#items = results.items()\n",
    "#print('%s items' % len(items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Can up number of returned results\n",
    "limit = 500\n",
    "cat = intake.open_stac_catalog(f'https://cmr.earthdata.nasa.gov/cmr-stac/ASF/collections?limit={limit}')\n",
    "col_info = pd.DataFrame(cat.metadata['collections'])\n",
    "print(len(col_info))\n",
    "col_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Opening item collection directly also not working\n",
    "# https://github.com/sat-utils/sat-stac/issues/65\n",
    "#from satstac import ItemCollection\n",
    "#col = 'C1595422627-ASF'\n",
    "#limit=10\n",
    "#url = f'https://cmr.earthdata.nasa.gov/cmr-stac/ASF/collections/{col}/items?limit={limit}'\n",
    "#print(url)\n",
    "#items = ItemCollection.open(url)\n",
    "#print(len(items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open an item directly (works but need to know item id in advance...)\n",
    "item = intake.open_stac_item('https://cmr.earthdata.nasa.gov/cmr-stac/ASF/collections/C1595422627-ASF/items/G1636018550-ASF')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# thumbnails don't always Nasa EarthData require authentication\n",
    "item.browse.plot.thumbnail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remote NetCDF + Authentication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# let's open this netcdf file directly (reads entire file into memory I think, not efficient or pretty, but works\n",
    "\n",
    "(username, account, password) = netrc.netrc().authenticators(\"urs.earthdata.nasa.gov\")\n",
    "fs = fsspec.filesystem('http', client_kwargs={'auth': aiohttp.BasicAuth(username, password)})\n",
    "\n",
    "with fs.open(item.data.urlpath) as f:\n",
    "    da = xr.open_dataset(f, group='/science/grids/data', engine='h5netcdf', chunks={})\n",
    "\n",
    "da"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "da['amplitude'].data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time \n",
    "\n",
    "# OR just download the file and work with it locally:\n",
    "localPath = item._stac_obj.download('data')\n",
    "da = xr.open_dataset(localPath, group='/science/grids/data', engine='h5netcdf', chunks={})\n",
    "da"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note that it is faster to download the entire netcdf file and open it up than to read remotely (lots of network requests) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remote Cloud-optimized geotiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item = intake.open_stac_item('https://cmr.earthdata.nasa.gov/cmr-stac/NSIDC_ECS/collections/C1908075185-NSIDC_ECS/items/G1921160945-NSIDC_ECS')\n",
    "#print(item.yaml())\n",
    "print(list(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(item['0'].yaml())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Assests with RasterIOSource Driver require these GDAL environment variables\n",
    "# and a properly configured .netrc with NASA EarthData credentials\n",
    "os.environ['GDAL_DISABLE_READDIR_ON_OPEN']='EMPTY_DIR'\n",
    "os.environ['GDAL_HTTP_COOKIEFILE']='.urs_cookies' \n",
    "os.environ['GDAL_HTTP_COOKIEJAR']='.urs_cookies'\n",
    "\n",
    "da = item['0'].to_dask()\n",
    "da"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NOTE that reading the cloud optimized geotiff remotely is very dast! only metadata is read to initialize"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}