Source code for dnppy.download.fetch_MODIS

__author__ = 'jwely'


from dnppy import core

from list_ftp import list_ftp
from list_http_e4ftl01 import list_http_e4ftl01
from download_url import download_url

import os

__all__ = ["fetch_MODIS"]

[docs]def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto, force_overwrite = False): """ Fetch MODIS Land products from one of two servers. If this function runs and downloads 0 files, check that your inputs are consistent with the naming convention at the appropriate server address. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org :param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1' :param version: version number, usually '004' or '041' or '005' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] NOTE: for some MODIS products, the h and v are omitted. :param outdir : output directory to save downloaded files :param start_dto: datetime object, the starting date of the range of data to download :param end_dto: datetime object, the ending date of the range of data to download :param force_overwrite: will re-download files even if they already exist :return out_filepaths: a list of filepaths to all files created by this function """ out_filepaths = [] # check formats tiles = core.enf_list(tiles) # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = _find_modis_product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates,_ = list_ftp(site, False, False, Dir) else: dates = list_http_e4ftl01(site) except: raise ValueError("Could not connect to {0}/{1}".format(site,Dir)) # refine contents down to just addresses of valid year and j_day good_dates = [] for date in dates: try: date_dto = datetime.strptime(date, "%Y.%m.%d") if start_dto <= date_dto <= end_dto: good_dates.append(date) except: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files,_ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address='/'.join(['ftp://'+site, Dir, good_date, afile]) else: address='/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) out_filepaths.append(outname) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print("Finished retrieving MODIS - {0} data!".format(product)) print("Downloaded {0} files".format(len(out_filepaths))) return out_filepaths
def _find_modis_product(product, version): """ Sub-function to determine server properties for MODIS data product. returns http/ftp handles the two current servers where aqua/terra MODIS data can be downloaded are site1='http://e4ftl01.cr.usgs.gov' site2='n5eil01u.ecs.nsidc.org' :param product: modis product such as 'MOD10A1' :param version: modis version, usually '005', '004', or '041' :returns site: server address where data can be found :returns ftp: ftp handle for open ftp session :returns Dir: subdirectory of server to further search for files of input product. """ sat_designation = product[0:3] prod_ID = product[3:] site1 = 'http://e4ftl01.cr.usgs.gov/' site2 = 'n5eil01u.ecs.nsidc.org' isftp = False Dir = False # refine the address of the desired data product if '10' in prod_ID: isftp = True site = site2 if sat_designation == 'MOD': if isftp: Dir = 'MOST/' + product + '.' + version else: site = site1+'MOLT/' + product + '.' + version elif sat_designation == 'MYD': if isftp: Dir = 'DP1/MOSA/' + product + '.' + version else: site = site1+'MOLA/' + product+'.' + version elif sat_designation == 'MCD': site = site1+'MOTA/' + product + '.' + version else: print('No such MODIS product is available for download with this script!') site = "None" return site, isftp, Dir # testing area if __name__ == "__main__": from datetime import datetime prod = "MOD11_L2" vers = "041" tile = ["1105", "1205"] outd = r"C:\Users\jwely\Desktop\troubleshooting\test\MOD10A1" star = datetime(2015,1,1) end = datetime(2015,1,3) fetch_MODIS(prod, vers, tile, outd, star, end)