#!/usr/bin/env slsh
require ("curl");
require ("pcre");

private variable Dir_Regexp = pcre_compile ("^d[-rwxtsr]+ .* ([-0-9A-Z_a-z]+)$");
private variable File_Regexp = pcre_compile ("^-[-rwxtsr]+ .* ([-0-9A-Z_a-z.]+)$");

private define write_to_string (sp, data)
{
   @sp = strcat (@sp, data);
   return 1;
}

private define write_to_fp (fp, data)
{
   variable len = bstrlen (data);
   if (bstrlen(data) != fwrite (data, fp))
     return -1;
   return 0;
}

private define get_dir_listing (c, dir)
{
   variable data = "";
   curl_setopt (c, CURLOPT_URL, path_concat (dir, ""));
   curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_string, &data);
   curl_perform (c);

   variable lines = strtok (data, "\r\n");
   variable dir_list = {};
   variable file_list = {};
   foreach (lines)
     {
	variable line = ();
	if (pcre_exec (Dir_Regexp, line))
	  {
	     list_append (dir_list, pcre_nth_substr (Dir_Regexp, line, 1));
	     continue;
	  }
	if (pcre_exec (File_Regexp, line))
	  {
	     list_append (file_list, pcre_nth_substr (File_Regexp, line, 1));
	     continue;
	  }
     }
   return dir_list, file_list;
}

private define find_obsid_url (c, rooturl, obsid);
private define find_obsid_url (c, rooturl, obsid)
{
   variable dirlist, filelist;
   (dirlist,) = get_dir_listing (c, rooturl);

   % If we are searching .../cat/, then don't recurse more.
   variable this_dir = path_basename (path_dirname (path_concat (rooturl,"")));
   variable recurse_ok = strncmp (this_dir, "cat", 3);
   
   foreach (dirlist)
     {
	variable subdir = ();
	variable dir = path_concat (rooturl, subdir);
	
	%() = fprintf (stdout, "%s\n", dir);

	if (subdir == obsid)
	  return dir;

	!if (recurse_ok)
	  continue;

	dir = find_obsid_url (c, dir, obsid);
	if (dir != NULL)
	  return dir;
     }
   return NULL;
}

private define get_recursive_listing (c, rooturl);
private define get_recursive_listing (c, rooturl, list)
{
   variable dirlist, filelist;
   (dirlist,filelist) = get_dir_listing (c, rooturl);

   foreach (filelist)
     {
	variable file = ();
	list_append (list, path_concat (rooturl, file));
     }

   foreach (dirlist)
     {
	variable subdir = ();
	get_recursive_listing (c, path_concat (rooturl, subdir), list);
     }
}

private define open_output_file (file)
{
   variable fp = fopen (file, "wb");
   if (fp != NULL)
     return fp;
   
   () = system (sprintf ("mkdir -p %s", path_dirname (file)));
   
   return fopen (file, "wb");
}
   
private define usage ()
{
   variable pgm = path_basename (__argv[0]);
   variable help = 
     ["Usage: $pgm [options] OBSID"$,
      "Options:",
      "  --help                   This help",
      "  --info                   Show files that would be downloaded",
      "  --cat=science|cal|er     Catalogues to search (default is all)",
      "  --dir=output-dir         Output directory (default obs_OBSID)",
      "  --download[=type-list]   Types of files to download",
      "",
      "Examples:",
      "Download JED's standard set of files for obs 105:",
      " $pgm --download 105"$,
      "",
      "Show what files would be downloaded from JED's standard set:",
      " $pgm --info --download 105"$,
      "",
      "Download the bias and pbk files for obs 105:",
      " $pgm --download=bias0,pbk0 105"$,
      "",
      "Download all files for obs 105:",
      " $pgm --download=all 105"$,
      " $pgm 105"$,
      "",
      "Download calibration obsid 61273 to cal/61273:",
      " $pgm --download --cat=cal,er --dir=cal/61273 61273"$,

      ""];

   foreach help (help)
     () = fprintf (stderr, "%s\n", help);
   exit (1);
}

private define parse_arg (arg)
{
   variable pos = is_substr (arg, "=");
   if (pos == 0)
     return (arg, NULL);
   variable value = substr (arg, pos+1, -1);
   arg = substr (arg, 1, pos-1);
   
   return arg, value;
}

define slsh_main ()
{
   variable i = 1;
   variable info_mode = 0;
   variable obsid = NULL;
   variable jed_set = "evt1,msk1,flt1,bpix1,stat1,asol1,mtl1,pbk0";
   variable set = NULL;
   variable catalogues = NULL;
   variable output_dir = NULL;
   
   while (i < __argc)
     {
	variable arg = __argv[i]; i++;
	variable value;

	(arg,value) = parse_arg (arg);

	if (arg == "--help")
	  usage ();
	if (arg == "--info")
	  {
	     info_mode = 1;
	     continue;
	  }
	if (arg == "--download")
	  {
	     if (value == NULL)
	       set = jed_set;
	     else 
	       {
		  set = strlow (value);
		  if (set == "all")
		    set = NULL;
	       }
	     continue;
	  }
	
	if (arg == "--cat")
	  {
	     if (value == NULL)
	       usage ();
	     catalogues = value;
	     continue;
	  }
	if (arg == "--dir")
	  {
	     if (value == NULL)
	       usage ();
	     output_dir = arg;
	     continue;
	  }

	obsid = arg;
	break;
     }
   if ((i != __argc) or (obsid == NULL))
     usage ();
   
   variable host = "cdaftp.harvard.edu";
   variable c, url, cat;

   if (catalogues == NULL)
     catalogues = "science,cal,er";

   foreach cat (strtok (catalogues, ","))
     {
	url = "ftp://$host/pub/$cat/"$;
	c = curl_new (url);

	%curl_setopt (c, CURLOPT_VERBOSE);
	url = find_obsid_url (c, url, obsid);
	if (url != NULL)
	  break;
     }
   if (url == NULL)
     {
	() = fprintf (stderr, "Unable to find the directory for obsid %s\n", obsid);
	exit (1);
     }

   variable files = {};
   get_recursive_listing (c, url, files);
   
   variable regexp = NULL;
   if (set != NULL)
     {
	regexp = sprintf (".*_(%s)\\.fits(\\.gz)?$", strtrans (strcompress (set, ","), ",", "|"));
	regexp = pcre_compile (regexp);
     }
   url = path_concat (url, "");   %  make sure final / is there
   variable len = strlen (url);

   if (info_mode == 0)
     {
	if (output_dir == NULL)
	  output_dir = "obs_${obsid}"$;
	() = mkdir (output_dir);
     }
   
   foreach (files)
     {
	variable file = ();
	variable filename = substr (file, len+1, -1);

	if (regexp != NULL)
	  {
	     if (0 == pcre_exec (regexp, filename))
	       continue;
	  }
	() = fprintf (stdout, "Downloading %s\n", filename);
	if (info_mode)
	  continue;
	
	filename = path_concat (output_dir, filename);
	variable fp = open_output_file (filename);
	if (fp == NULL)
	  {
	     () = fprintf (stderr, "Unable to open %s\n", filename);
	     exit (1);
	  }
	curl_setopt (c, CURLOPT_WRITEFUNCTION, &write_to_fp, fp);
	curl_setopt (c, CURLOPT_URL, file);
	curl_perform (c);
	if (-1 == fclose (fp))
	  {
	     () = fprintf (stderr, "Unable to close %s -- disk full?\n", filename);
	     exit (1);
	  }
     }
}