// $Id: CatalogGen.java,v 1.12 2004/11/30 23:07:49 edavis Exp $

package thredds.cataloggen;

import thredds.catalog.*;
import thredds.cataloggen.config.CatGenConfigMetadataFactory;
import thredds.cataloggen.config.CatalogGenConfig;
import thredds.cataloggen.config.DatasetSource;

import java.io.PrintStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Iterator;
import java.util.ArrayList;

import org.apache.log4j.*;

/**
 * CatalogGen crawls dataset sources given in a CatalogGenConfig file
 * to produce THREDDS catalogs.
 *
 * To generate a catalog from a config file:
 * <pre>
 *   String inFileName = "file:/home/edavis/testCatGenConfig.xml";
 *   String outFileName = "/home/edavis/testCatGenConfig-results.xml";
 *   StringBuffer log = new StringBuffer();
 *   CatalogGen catGen = new CatalogGen( inFileName);
 *   if ( catGen.isValid( log))
 *   {
 *     catGen.expand();
 *     catGen.writeCatalog( outFileName);
 *   }
 * </pre>
 *
 *
 * @author Ethan Davis
 * @version $Ver$
 */
public class CatalogGen
{
  private static Logger logger = Logger.getLogger( CatalogGen.class.getName());

  /** The name of the config file. */
  private URL configDocURL = null;
  /** The catalog: initially as a CatGen config file, until expanded. */
  private InvCatalog catalog = null;

  /** The catalog factory that knows about CatalogGenConfig metadata. */
  private InvCatalogFactory catFactory = null;

  /**
   * Constructs the CatalogGen for the given config document.
   *
   * @param configDocURL - the URL of the configuration document
   */
  public CatalogGen( URL configDocURL)
  {
    // Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
    logger.debug( "CatalogGen(URL): create catalog and CatalogGenConfig converter." );
    this.catFactory = new InvCatalogFactory( "default", true);
    this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
                                             new CatGenConfigMetadataFactory());

    this.configDocURL = configDocURL;

    // Read the given XML config file.
    logger.debug( "CatalogGen(URL): reading the config doc <" + this.configDocURL.toString() + ">.");
    this.catalog = this.catFactory.readXML( this.configDocURL.toString());
    logger.debug( "CatalogGen(URL): done.");
  }

  /**
   * Constructs the CatalogGen for the given config document InputStream.
   *
   * @param configDocInputStream - the InputStream from which to read the config document.
   * @param configDocURL - the URL for the config document.
   */
  public CatalogGen( InputStream configDocInputStream, URL configDocURL )
  {
    // Create a InvCatalogFactory with CATALOG_GEN_CONFIG MetadataType registered.
    logger.debug( "CatalogGen(InputStream): create catalog and CatalogGenConfig converter." );
    this.catFactory = new InvCatalogFactory( "default", true );
    this.catFactory.registerMetadataConverter( MetadataType.CATALOG_GEN_CONFIG.toString(),
                                               new CatGenConfigMetadataFactory() );

    this.configDocURL = configDocURL;
    // Read the given XML config file.
    logger.debug( "CatalogGen(InputStream): reading the config doc <" + configDocURL.toString() + ">." );
    this.catalog = this.catFactory.readXML( configDocInputStream, URI.create( configDocURL.toExternalForm()) );
    logger.debug( "CatalogGen(InputStream): CatalogGenConfig doc <" + this.catalog.getName() + "> read.");

  }

  /**
   * Checks the validity of the configuration file.
   * @param out - a StringBuffer with validity error and warning messages.
   * @return - true if no errors, false if errors exist
   */
  public boolean isValid( StringBuffer out)
  {
    logger.debug( "isValid(): start");
    return( this.catalog.check( out));
  }

  /**
   * Expand the catalog. Each of the CatalogGenConfig metadata elements
   * is expanded into its constituent datasets.
   */
  public void expand()
  {
    CatalogGenConfig tmpCgc = null;
    List cgcList = null;
    List dssList = null;
    DatasetSource dss = null;

    // Find and loop through each CatGenConfigMetadata object.
    logger.debug( "expand(): find all CatGenConfigMetadata elements.");
    List mdataList = findCatGenConfigMdata( this.catalog.getDatasets());
    for ( int i = 0; i < mdataList.size(); i++)
    {
      InvMetadata mdata = (InvMetadata) mdataList.get( i);
      logger.debug( "expand(): <# " + i + "> metadata element of type CatalogGenConfig." );
      cgcList = (List) mdata.getContentObject();

      // *****
      // @todo Move this code into CatalogGenConfig.expand().
      // *****
      // Loop through the CatalogGenConfig objects in current InvMetadata.
      for ( int j = 0; j < cgcList.size(); j++)
      {
        tmpCgc = (CatalogGenConfig) cgcList.get( j);
        logger.debug( "expand(): <# " + j + "> catalogGenConfig element." );
        dss = tmpCgc.getDatasetSource();
        dss.fullExpand();
      }
//      logger.debug( "expand(): List datasets that are siblinks of current metadata record CGCM(" + i + ").");
//      List list = mdata.getParentDataset().getDatasets();
//      Iterator it = list.iterator();
//      while ( it.hasNext())
//      {
//        InvDatasetImpl curDs = (InvDatasetImpl) it.next();
//        logger.debug( "Dataset URL is " + curDs.getUrlPath() + ".");
//        logger.debug( "Dataset name is " + curDs.getName() + ".");
//      }

      // Remove the current metadata element from its parent dataset.
      logger.debug( "expand(): Remove metadata record CGCM(" + i + ")." );
      ( (InvDatasetImpl) mdata.getParentDataset()).removeLocalMetadata( mdata);
      // *****

      // Finish this catalog now that done building it.
      ((InvCatalogImpl) this.catalog).finish();
    }
  }

  /**
   * Writes the catalog as XML. The catalog is written to the file given
   * in <tt>outFileName</tt>. If <tt>outFileName</tt> is null, the catalog
   * is written to standard out.
   *
   * @param outFileName - the pathname of the output file.
   */
  public boolean writeCatalog( String outFileName)
  {
    logger.debug( "writeCatalog(): writing catalog to " + outFileName + ".");

    String invCatDTD = "http://www.unidata.ucar.edu/projects/THREDDS/xml/InvCatalog.0.6.dtd";
    logger.debug( "writeCatalog(): set the catalogs DTD (" + invCatDTD + ").");
    // Set the catalogs DTD.
    ( (InvCatalogImpl) catalog).setDTDid( invCatDTD);

    // Print the catalog as an XML document.
    if ( outFileName == null)
    {
      try
      {
        logger.debug( "writeCatalog(): write catalog to System.out.");
        this.catFactory.writeXML( (InvCatalogImpl) catalog, System.out);
      }
      catch ( java.io.IOException e)
      {
        logger.debug( "writeCatalog(): exception when writing to stdout.\n" +
                e.toString());
        //e.printStackTrace();
        return( false);
      }
      return( true);
    }
    else
    {
      logger.debug( "writeCatalog(): try writing catalog to the output file (" + outFileName + ").");
      if ( this.catFactory.writeXML( (InvCatalogImpl) catalog, outFileName))
      {
        logger.debug( "writeCatalog(): catalog written to " + outFileName + ".");
        return( true);
      }
      else
      {
        logger.debug( "writeCatalog(): catalog not written to " + outFileName + ".");
        return( false);
      }
    }
  }

  protected boolean compareCatalog( InvCatalogImpl expectedCatalog)
  {
    return( ( (InvCatalogImpl) this.catalog).equals( expectedCatalog));
  }

  InvCatalog getCatalog() { return( this.catalog ); }

  private List findCatGenConfigMdata( List datasets)
  {
    List mdataList = new ArrayList();
    if ( datasets == null) return( mdataList );

    // Iterate through list of datasets.
    Iterator it = datasets.iterator();
    InvDataset curDataset = null;
    while ( it.hasNext() )
    {
      curDataset = (InvDataset) it.next();

      // Get all the local metadata for the given dataset.
      ThreddsMetadata tm = ((InvDatasetImpl) curDataset).getLocalMetadata();

      // Iterate over the local InvMetadata checking for CatalogGenConfig metadata.
      Iterator itMdata = tm.getMetadata().iterator();
      InvMetadata curMetadata = null;
      while ( itMdata.hasNext())
      {
        curMetadata = (InvMetadata) itMdata.next();
        if ( (curMetadata.getMetadataType() != null) &&
             curMetadata.getMetadataType().equals( MetadataType.CATALOG_GEN_CONFIG.toString() ) )
        {
          mdataList.add( curMetadata );
        }
        else if ( (curMetadata.getNamespaceURI() != null) &&
                  curMetadata.getNamespaceURI().equals( CatalogGenConfig.CATALOG_GEN_CONFIG_NAMESPACE_URI_0_5 ))
        {
          mdataList.add( curMetadata );
        }
      }

      // Recurse through nested datasets and find CatalogGenConfig metadata.
      mdataList.addAll( this.findCatGenConfigMdata( curDataset.getDatasets() ) );
    }

    return( mdataList);
  }

  public static void main(String[] args)
  {
    String configFileName = null;
    File configFile = null;
    URL configDocURL = null;

    String logFileName = null;
    File logFile = null;
    String logLevelName = null;
    Level logLevel = null;
    PatternLayout layout = new PatternLayout("%d{yyyy-MM-dd HH:mm:ss.SSS} [%10r] %-5p - %c - %m%n");
    Appender appender = null;
    Logger catGenLogger = null;

    String outFileName = null;

    CatalogGen catGen = null;

    StringBuffer log = new StringBuffer();

    /* @todo Add ability to translate GDS catalog into THREDDS catalog. */
    // Deal with command line arguments.
    if (args.length == 1)
    {
      if ( args[ 0].equals( "-help"))
      {
        usage( System.out);
        System.exit( 0);
      }
      else
      {
        // Only the input Doc URL is given.
        configFileName = args[ 0];
      }
    }
    else if ( args.length == 2)
    {
      // Both the input Doc URL and the output file name are given.
      configFileName = args[ 0];
      outFileName = args[ 1];
    }
    else if ( args.length == 4 || args.length == 5)
    {
      if ( args[0].equals( "-log"))
      {
        logFileName = args[1];
        logLevelName = args[2];
        configFileName = args[3];
        if ( args.length == 5 )
        {
          outFileName = args[4];
        }

        logLevel = Level.toLevel( logLevelName);
        logFile = new File( logFileName);
        try
        {
          appender = new RollingFileAppender( layout, logFile.toString(), false);
        }
        catch ( IOException e )
        {
          System.err.println( "CatalogGen: Error - log file creation got IOException (" + logFile.toString() + ")");
          System.err.println( "    " + e.getMessage());
          System.exit( 1);
        }

      }
      else
      {
        System.err.println( "CatalogGen: Did not understand your request (1).");
        usage( System.err);
        System.exit( 1);
      }
    }
    else
    {
      System.err.println( "CatalogGen: Did not understand your request (2).");
      usage( System.err);
      System.exit( 1);
    }

    // Setup logging and log level.
    if ( appender == null)
    {
      appender = new ConsoleAppender( layout);
      logLevel = Level.toLevel( "OFF");
    }
    catGenLogger = Logger.getLogger( "thredds.cataloggen");
    catGenLogger.setLevel( logLevel );

    catGenLogger.addAppender( appender);


    // Create a URL for the configuration document.
    try
    {
      URI tmpURI;
      if ( ! configFileName.startsWith( "http:") && ! configFileName.startsWith( "file:"))
      {
        configFile = new File( configFileName);
        if ( ! configFile.canRead())
        {
          logger.error( "CatalogGen: config file is not readable (" + configFile.toString() + ")");
          System.err.println( "CatalogGen: config file is not readable (" + configFile.toString() + ")");
          System.exit( 1);
        }
        tmpURI = configFile.toURI();
      }
      else
      {
        tmpURI = new URI( configFileName);
      }
      configDocURL = tmpURI.toURL();
    }
    catch ( MalformedURLException e )
    {
      logger.error( "CatalogGen: The config doc path is not a valid URL (" + e.getMessage() + ")." );
      System.err.println( "CatalogGen: The config doc path is not a valid URL." );
      usage( System.err);
      System.exit( 1);
    }
    catch ( URISyntaxException e )
    {
      logger.error( "CatalogGen: The config doc path is not a valid URI (" + e.getMessage() + ")." );
      System.err.println( "CatalogGen: The config doc path is not a valid URI." );
      usage( System.err);
      System.exit( 1);
    }

    //
    catGen = new CatalogGen( configDocURL);
    if ( catGen.isValid( log))
    {
      catGen.expand();
      if ( catGen.writeCatalog( outFileName))
      {
        logger.debug( "CatalogGen: the catalog was written to " + outFileName + ".");
        System.out.println( "CatalogGen: the catalog was written to " + outFileName + ".");
        System.exit( 0);
      }
      else
      {
        logger.error( "CatalogGen: the catalog was not written to " + outFileName + ".");
        System.err.println( "CatalogGen: the catalog was not written to " + outFileName + ".");
        System.exit( 1);
      }
    }
    else
    {
      logger.error( "CatalogGen: Invalid config file (" + configFileName + "):\n" +
              log.toString());
      System.err.println( "CatalogGen: Invalid config file (" + configFileName + "):");
      System.err.println( log.toString());
      System.exit( 1);
    }
  }

  private static void usage( PrintStream ps)
  {
    ps.println( "Usage:");
    ps.println( "  CatalogGen [options] <configDocName> [<outFileName>]\n" +
                "    Given a CatalogGenConfig 0.5 document, produce a completed\n" +
                "    InvCatalog 0.6 document. NOTE: the configuration document may\n" +
                "    be specified as a local file name or as a URL.");
    ps.println( "Options:\n" +
                "  -help\n" +
                "      Print this usage message.\n" +
                "  -log <logFileName> <logLevel>\n" +
                "      Write a log file at the given log level (OFF, FATAL, WARN, INFO, DEBUG, ALL).\n");
  }

}
/*
 * $Log: CatalogGen.java,v $
 * Revision 1.12  2004/11/30 23:07:49  edavis
 * Update for new DatasetSource API.
 *
 * Revision 1.11  2004/06/03 20:15:54  edavis
 * Modify findCatGenConfigMdata() to work with new catalog object model. Add
 * a constructor that reads from an InputStream.
 *
 * Revision 1.10  2004/05/11 20:13:12  edavis
 * Update for changes to thredds.catalog data model (still InvCat 0.6).
 *
 * Revision 1.9  2004/03/05 06:28:09  edavis
 * Fix the usage message.
 *
 * Revision 1.8  2003/09/24 19:35:17  edavis
 * Fix how output file is handled.
 *
 * Revision 1.7  2003/09/23 21:17:55  edavis
 * Allow configuration document to be specified as a local file path, as a local
 * file URL ("file:"), or as an HTTP URL ("http:").
 *
 * Revision 1.6  2003/09/05 22:01:33  edavis
 * Change some logging from debug() to info().
 *
 * Revision 1.5  2003/08/29 21:41:46  edavis
 * The following changes where made:
 *
 *  1) Added more extensive logging (changed from thredds.util.Log and
 * thredds.util.Debug to using Log4j).
 *
 * 2) Improved existing error handling and added additional error
 * handling where problems could fall through the cracks. Added some
 * catching and throwing of exceptions but also, for problems that aren't
 * fatal, added the inclusion in the resulting catalog of datasets with
 * the error message as its name.
 *
 * 3) Change how the CatGenTimerTask constructor is given the path to the
 * config files and the path to the resulting files so that resulting
 * catalogs are placed in the servlet directory space. Also, add ability
 * for servlet to serve the resulting catalogs.
 *
 * 4) Switch from using java.lang.String to using java.io.File for
 * handling file location information so that path seperators will be
 * correctly handled. Also, switch to java.net.URI rather than
 * java.io.File or java.lang.String where necessary to handle proper
 * URI/URL character encoding.
 *
 * 5) Add handling of requests when no path ("") is given, when the root
 * path ("/") is given, and when the admin path ("/admin") is given.
 *
 * 6) Fix the PUTting of catalogGenConfig files.
 *
 * 7) Start adding GDS DatasetSource capabilities.
 *
 * Revision 1.4  2003/07/03 20:38:28  edavis
 * When made DatasetSource an abstract class, changed resolve() to expand().
 *
 * Revision 1.3  2003/03/18 21:10:02  edavis
 * Updated the usage message.
 *
 * Revision 1.2  2003/03/04 23:02:23  edavis
 * Lots of changes made to support CatGenServlet.
 *
 *
 */
