// $Id: NcMLReader.java,v 1.5 2004/12/10 19:44:48 caron Exp $
/*
 * Copyright 1997-2000 Unidata Program Center/University Corporation for
 * Atmospheric Research, P.O. Box 3000, Boulder, CO 80307,
 * support@unidata.ucar.edu.
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
 * General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package ucar.nc2.ncml;

import ucar.nc2.*;
import ucar.nc2.dataset.*;
import ucar.nc2.util.CancelTask;
import ucar.unidata.util.StringUtil;
import ucar.ma2.Array;
import ucar.ma2.Index;
import ucar.ma2.IndexIterator;

import org.jdom.*;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;

import java.io.*;
import java.net.*;
import java.util.*;

/**
 * Read NcML and create NetcdfDataset.
 * Note that this is thread-safe.
 *
 * @see ucar.nc2.NetcdfFile
 * @author caron
 * @version $Revision: 1.5 $ $Date: 2004/12/10 19:44:48 $
 */

public class NcMLReader {
  private static final Namespace ncNS = Namespace.getNamespace("nc", XMLEntityResolver.NJ22_NAMESPACE);
  private static boolean debugURL = false, debugXML = false, showParsedXML = false;
  private static boolean debugOpen = false, debugConstruct = false, debugCmd = false, debugReletiveURI = false;
  private static boolean debugAgg = false, debugAggDetail = false;

  static public void setDebugFlags( ucar.nc2.util.DebugFlags debugFlag) {
    debugURL =  debugFlag.isSet("NcML/debugURL");
    debugXML =  debugFlag.isSet("NcML/debugXML");
    showParsedXML =  debugFlag.isSet("NcML/showParsedXML");
    debugReletiveURI =  debugFlag.isSet("NcML/debugReletiveURI");
    debugOpen =  debugFlag.isSet("NcML/debugOpen");
    debugConstruct =  debugFlag.isSet("NcML/debugConstruct");
    debugAgg =  debugFlag.isSet("NcML/debugAgg");
    debugAggDetail =  debugFlag.isSet("NcML/debugAggDetail");
  }

  static private boolean validate = false;

  // use NCML to augment the dataset; getting it as a resource stream
  static public void wrapNcMLresource(NetcdfDataset ncDataset, String ncmlResourceLocation, CancelTask cancelTask) throws IOException {
    ClassLoader cl = ncDataset.getClass().getClassLoader();
    InputStream is = cl.getResourceAsStream(ncmlResourceLocation);
    if (is == null)
      throw new FileNotFoundException(ncmlResourceLocation);

    // ncml modifies dataset
    NcMLReader reader = new NcMLReader();
    reader.wrapNcML(is, ncmlResourceLocation, ncDataset, cancelTask);
  }


  // use NCML to augment the dataset; opening as a regular URL
  static public void wrapNcML(NetcdfDataset ncDataset, String ncmlLocation, CancelTask cancelTask) throws IOException {
    NcMLReader reader = new NcMLReader();

    // ncml modifies dataset
    reader.wrapNcML(ncmlLocation, ncDataset, cancelTask);
  }

  /**
   * Read an NcML file from a URL location, and construct a NetcdfDataset.
   * @param ncmlLocation the URL location string of the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   */
  public NetcdfDataset readNcML(String ncmlLocation, CancelTask cancelTask) throws IOException, java.net.MalformedURLException {
    return readNcML( ncmlLocation, null, cancelTask);
  }

  /**
   * Read an NcML file from a URL location, and construct a NetcdfDataset.
   * @param ncmlLocation the URL location string of the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   */
  public NetcdfDataset readNcML(String ncmlLocation, String referencedDatasetUri, CancelTask cancelTask) throws IOException, java.net.MalformedURLException {
    URL url =  new URL( ncmlLocation);

    if (debugURL) {
      System.out.println(" NcMLReader open "+ncmlLocation);
      System.out.println("   URL = "+url.toString());
      System.out.println("   external form = "+url.toExternalForm());
      System.out.println("   protocol = "+url.getProtocol());
      System.out.println("   host = "+url.getHost());
      System.out.println("   path = "+url.getPath());
      System.out.println("  file = "+url.getFile());
    }

    org.jdom.Document doc = null;
    try {
      SAXBuilder builder = new SAXBuilder();
      if (debugURL) System.out.println(" NetcdfDataset URL = <"+url+">");
      doc = builder.build(url);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML) System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println ("*** NetcdfDataset/showParsedXML = \n"+xmlOut.outputString(doc)+"\n*******");
    }

    Element netcdfElem = doc.getRootElement();

    if (referencedDatasetUri == null) {
    // the ncml probably refers to another dataset, but doesnt have to
      referencedDatasetUri = netcdfElem.getAttributeValue("location");
      if (referencedDatasetUri == null)
        referencedDatasetUri = netcdfElem.getAttributeValue("uri");
    }

    NetcdfDataset ncd = readNetcdf( ncmlLocation, referencedDatasetUri, netcdfElem, cancelTask);
    if (debugOpen) System.out.println("***NcMLReader.readNetcdf result= \n"+ncd);
    return ncd;
  }

  /**
    * Read NcML doc from an InputStream, and construct a NetcdfDataset.
    * @param ins the InputStream containing the NcML document
    * @param cancelTask allow user to cancel the task; may be null
    */
   public NetcdfDataset readNcML(InputStream ins, CancelTask cancelTask) throws IOException, java.net.MalformedURLException {

     org.jdom.Document doc = null;
     try {
       SAXBuilder builder = new SAXBuilder();
       doc = builder.build(ins);
     } catch (JDOMException e) {
       throw new IOException(e.getMessage());
     }
     if (debugXML) System.out.println(" SAXBuilder done");

     if (showParsedXML) {
       XMLOutputter xmlOut = new XMLOutputter();
       System.out.println ("*** NetcdfDataset/showParsedXML = \n"+xmlOut.outputString(doc)+"\n*******");
     }

     Element netcdfElem = doc.getRootElement();

     // the ncml probably refers to another dataset, but doesnt have to
     String referencedDatasetUri = netcdfElem.getAttributeValue("location");
     if (referencedDatasetUri == null)
       referencedDatasetUri = netcdfElem.getAttributeValue("uri");

     NetcdfDataset ncd = readNetcdf( null, referencedDatasetUri, netcdfElem, cancelTask);
     if (debugOpen) System.out.println("***NcMLReader.readNetcdf result= \n"+ncd);
     return ncd;
   }

  /**
   * Read an NcML file from a URL location, and construct a NetcdfDataset.
   * @param ncmlLocation the URL location string of the NcML document
   * @param cancelTask allow user to cancel the task; may be null
   */
  NetcdfDataset readNetcdf( String ncmlLocation, String referencedDatasetUri,
                            Element netcdfElem, CancelTask cancelTask) throws IOException {

   // deal with reletive URL
    if ((ncmlLocation != null) && (referencedDatasetUri != null)) {
      URI ncmlURI = URI.create(ncmlLocation);
      URI refURI = URI.create(referencedDatasetUri);
      URI resolvedURI = ncmlURI.resolve(refURI);
      if (debugReletiveURI) {
        System.out.println("ncmlURI= "+ncmlURI.isOpaque()+" "+ncmlURI.isAbsolute() +" "+ncmlURI);
        System.out.println("refURI= "+refURI.isOpaque()+" "+refURI.isAbsolute()+" "+refURI);
        System.out.println("resolvedURI= "+resolvedURI.isOpaque()+" "+resolvedURI.isAbsolute()+" "+resolvedURI);
      }
      // LOOK this wont resolve reletive file: URLs
      referencedDatasetUri = resolvedURI.toASCIIString();
    }

    // common error causing infinite regression
    if ((referencedDatasetUri != null) && referencedDatasetUri.equals(ncmlLocation))
      throw new IllegalStateException("NcML location attribute refers to the NcML document itself"+referencedDatasetUri);


    // open the referenced dataset
    NetcdfDataset refds = null;
    if (referencedDatasetUri != null) {
      refds = NetcdfDataset.openDataset( referencedDatasetUri, false, cancelTask);
    }

    Element elemE = netcdfElem.getChild("explicit", ncNS);
    boolean isExplicit = (elemE != null);

    NetcdfDataset newds;
    if (isExplicit || (refds == null)) {
      newds = new NetcdfDataset();
      if (refds == null)
        refds = newds;

    } else { // modify
      newds = refds;
    }

    readNetcdf( ncmlLocation, newds, refds, netcdfElem, cancelTask);
    return newds;
  }

  /**
   * Read an NcML file from a URL location, and wrap the given NetcdfDataset with it.
   * @param ncmlLocation the URL location string of the NcML document
   * @param refds refers to this dataset (override anything in the ncml)
   * @param cancelTask allow user to cancel the task; may be null
   */
  public void wrapNcML( String ncmlLocation, NetcdfDataset refds, CancelTask cancelTask) throws IOException {

    org.jdom.Document doc = null;
    try {
      SAXBuilder builder = new SAXBuilder();
      if (debugURL) System.out.println(" NetcdfDataset URL = <"+ncmlLocation+">");
      doc = builder.build(ncmlLocation);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML) System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println ("*** NetcdfDataset/showParsedXML = \n"+xmlOut.outputString(doc)+"\n*******");
    }

    Element netcdfElem = doc.getRootElement();

    readNetcdf( ncmlLocation, refds, refds, netcdfElem, cancelTask);
    if (debugOpen) System.out.println("***NcMLReader.wrapNcML result= \n"+refds);
  }

  /**
   * Read an NcML file from a URL location, and wrap the given NetcdfDataset with it.
   * @param ncmlLocation the URL location string of the NcML document, for debugging. The location of the dataset
   *   is taken from refds.
   * @param refds refers to this dataset (override anything in the ncml)
   * @param cancelTask allow user to cancel the task; may be null
   */
  public void wrapNcML( InputStream is, String ncmlLocation, NetcdfDataset refds, CancelTask cancelTask) throws IOException {

    org.jdom.Document doc = null;
    try {
      SAXBuilder builder = new SAXBuilder();
      if (debugURL) System.out.println(" NetcdfDataset URL = <"+ncmlLocation+">");
      doc = builder.build(is);
    } catch (JDOMException e) {
      throw new IOException(e.getMessage());
    }
    if (debugXML) System.out.println(" SAXBuilder done");

    if (showParsedXML) {
      XMLOutputter xmlOut = new XMLOutputter();
      System.out.println ("*** NetcdfDataset/showParsedXML = \n"+xmlOut.outputString(doc)+"\n*******");
    }

    Element netcdfElem = doc.getRootElement();

    readNetcdf( refds.getLocation(), refds, refds, netcdfElem, cancelTask);
    if (debugOpen) System.out.println("***NcMLReader.wrapNcML result= \n"+refds);
  }

  private void readNetcdf( String ncmlLocation, NetcdfDataset newds, NetcdfDataset refds, Element netcdfElem, CancelTask cancelTask) throws IOException {

    //                    refds != null               refds == null
    //  explicit            explicit                      new (ref=new)
    //  readMeatadata       modify (new=ref)              new (ref=new)
    //
    //  see if its an explicit only

    if (debugOpen) System.out.println("NcMLReader.readNetcdf ncml= "+ncmlLocation+" referencedDatasetUri= "+refds.getLocation());
    newds.setLocation( ncmlLocation);
    newds.setId( netcdfElem.getAttributeValue("id"));
    newds.setTitle( netcdfElem.getAttributeValue("title"));
    if (refds != null)
      newds.setReferencedDatasetUri( refds.getLocation());

     // the root group
    readGroup( newds, refds, null, null, netcdfElem);

    // read in nested netcdf elements : "union" aggregation
    Aggregation aggUnion = null;
    java.util.List ncList = netcdfElem.getChildren("netcdf", ncNS);
    for (int j=0; j< ncList.size(); j++) {
      if (aggUnion == null) {
        aggUnion = new Aggregation("union");
        newds.setAggregation( aggUnion);
      }

      Element netcdfElemNested = (Element) ncList.get(j);
      String referencedDatasetUriNested = netcdfElemNested.getAttributeValue("location");
      if (referencedDatasetUriNested == null)
        referencedDatasetUriNested = netcdfElemNested.getAttributeValue("uri");

      NetcdfDataset nestedDs = readNetcdf( ncmlLocation, referencedDatasetUriNested, netcdfElemNested, cancelTask);
      aggUnion.addDataset( nestedDs);
      transferDataset(nestedDs, newds);
      if (debugAggDetail) System.out.println(" debugAgg: nested dataset = "+nestedDs);
    }

    // joinExisting or joinNew aggregations
    Element aggElem = netcdfElem.getChild("aggregation", ncNS);
    if (aggElem != null) {
      Aggregation agg = readAgg(aggElem, cancelTask);
      newds.setAggregation( agg);

      if (agg.isJoinNew)
        aggNewDimension( newds, agg, cancelTask);
      else
        aggExistingDimension( newds, agg, cancelTask);
    }

    newds.finish();
  }

/*        // look for attributes
    java.util.List attList = netcdfElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      ucar.nc2.Attribute a = readAtt((Element) attList.get(j));
      ds.addAttribute( rootGroup, a);
    }

    // look for dimensions
    java.util.List dimList = netcdfElem.getChildren("dimension", ncNS);
    for (int j=0; j< dimList.size(); j++) {
      Dimension d = readDim((Element) dimList.get(j));
      ds.addDimension( rootGroup, d);
      if (debug) System.out.println(" add dim = "+d);
    }

    // look for coordAxis
    java.util.List axisList = netcdfElem.getChildren("coordinateAxis", ncNS);
    for (int j=0; j< axisList.size(); j++) {
      VariableDS v = readCoordAxis(ds, rootGroup, null, (Element) axisList.get(j));
      ds.addVariable( rootGroup, v);
      ds.addCoordinateAxis( v);
      if (debug) System.out.println(" add coordAxis = "+v.getName());
    }

    // look for variables
    java.util.List varList = netcdfElem.getChildren("variable", ncNS);
    for (int j=0; j< varList.size(); j++) {
      VariableDS v = readVariable(ds, rootGroup, null, (Element) varList.get(j));
      ds.addVariable( rootGroup, v);
      if (debug) System.out.println(" add var = "+v.getName());
    }

    // look for structures
    java.util.List structList = netcdfElem.getChildren("structure", ncNS);
    for (int j=0; j< structList.size(); j++) {
      StructureDS s = readStructure(ds, rootGroup, null, (Element) structList.get(j));
      ds.addVariable( null, s);
      if (debug) System.out.println(" add struct = "+s.getName());
    }

    // look for groups
    java.util.List groupList = netcdfElem.getChildren("group", ncNS);
    for (int j=0; j< groupList.size(); j++) {
      Group g = readGroup(ds, rootGroup, (Element) groupList.get(j));
      ds.addGroup( rootGroup, g);
      if (debug) System.out.println(" add group = "+g.getName());
    }

     // process readMetadata command
     Element readElem = netcdfElem.getChild("readMetadata", ncNS);
     if (readElem != null) {
       readMetadata(ds, readElem, cancelTask);
     }

     // look for coordinate transforms
     java.util.List list = netcdfElem.getChildren("coordinateTransform", ncNS);
     for (int j=0; j< list.size(); j++) {
       CoordinateTransform ct = readCoordTransform(ds, (Element) list.get(j));
       ds.addCoordinateTransform( ct);
       if (debug) System.out.println(" add coordinateTransforms = "+ct.getName());
     }

     // look for coordinate systems
     list = netcdfElem.getChildren("coordinateSystem", ncNS);
     for (int j=0; j< list.size(); j++) {
       CoordinateSystem cs = readCoordSystem(ds, (Element) list.get(j));
       ds.addCoordinateSystem( cs);
       if (debug) System.out.println(" add coordinateSystem = "+cs.getName());
     }

    /* copy parent objects into this dataset LOOK ??????
    if (parent != null) {
      if (debugDetail) System.out.println(" copy from parent:");
      addElementsCopy( parent, ds);
    } */

    /*  read in nested netcdf elements
    java.util.List ncList = netcdfElem.getChildren("netcdf", ncNS);
    for (int j=0; j< ncList.size(); j++) {
      NetcdfDataset nestedDs = new NetcdfDataset();
      Element netcdfElemNested = (Element) ncList.get(j);
      String uriStringNested = netcdfElemNested.getAttributeValue("uri");
      readNetcdf(ds, nestedDs, netcdfElemNested, uriStringNested);
      ds.nestedDatasets.add( nestedDs);
      if (debugAggDetail) System.out.println(" debugAgg: nested dataset = "+nestedDs);
    } */

    /* process aggregation commands
    ArrayList aggEVars = new ArrayList();
    ArrayList aggNew = new ArrayList();
    java.util.List aggList = netcdfElem.getChildren("aggregation", ncNS);
    for (int j=0; j< aggList.size(); j++) {
      Aggregation agg = readAgg((Element) aggList.get(j));
      if (agg.type.equals("joinExisting"))
        doJoinExistingAgg( ds, agg, aggEVars);
      else if (agg.type.equals("joinNew"))
        aggNew.add( agg);
    } */

    /* copy of variables-dont want variables from nested datasets
    ArrayList myvars = new ArrayList();

    // copy child objects to parent
    // must have done all nested datasets before copying back to parent
    // note this is copy, not copyReplace
    ArrayList nestedDatasets = ds.nestedDatasets();
    for (int j=0; j< nestedDatasets.size(); j++) {
      NetcdfDataset nestedDs = (NetcdfDataset) nestedDatasets.get(j);
      if (debugDetail) System.out.println(" copy from children:");
      //addElements( nestedDs, ds);
    } */

    /* now we can create the real aggExisting variables
    for (int i=0; i<aggEVars.size(); i++) {
      VariableDS v = (VariableDS) aggEVars.get(i);
      VariableAggExisting vagg = new VariableAggExisting( ds, v);
      ds.variables().remove( v); // remove previous declaration, if any
      ds.variables().add( vagg);
    }

    // now we can process the aggNew
    for (int i=0; i<aggNew.size(); i++) {
      Aggregation agg = (Aggregation) aggNew.get(i);
      doJoinNewAgg( ds, agg);
    } */

    /* look for dimension values (joinNew aggregation)
    Element dvElem = netcdfElem.getChild("dimensionValue", ncNS);
    if (dvElem != null) {
      String dimName = dvElem.getAttributeValue("name");
      ds.setAggDimensionName(dimName);
      ds.setAggDimensionValue(dvElem.getAttributeValue("value"));

      // remove any dimension of this name
      Dimension aggDim = ds.findDimension( dimName);
      if (null != aggDim)
        ds.dimensions().remove( aggDim);

      // remove any variable dimension of this name
      // at this point, the dimensions are still encoded in the shapeS string
      ArrayList vars = ds.variables();
      for (int i=0; i<vars.size(); i++) {
        VariableDS v = (VariableDS) vars.get(i);
        v.removeDimension(dimName);
      }
    } */

    /* reset variable dimensions, derive shapes, etc
    // only on "myvars"
    for (int i=0; i<myvars.size(); i++) {
      VariableDS v = (VariableDS) myvars.get(i);
      v.finish();
      if (debugDetail) System.out.println(" finish var = "+v.getName());
    } */

    // find any type1 and type2 aggregation variables
    //convertAggNew(ds);
    //convertAggExisting(ds);

    /* last thing is to parse conventions : LOOK not sure about nested case
    Element pcElem = netcdfElem.getChild("parseConventions", ncNS);
    if (pcElem != null) {
      NetcdfDataset aug = ucar.nc2.dataset.conv.Convention.factory( ds);
      if (aug != null) ds = aug;
    }

    //System.out.println("End of factory parsing "+ds.getUrlString()+" ==\n"+ds+"\n");
   }

    /* read a dataset element
  protected void readNetcdf( NetcdfDataset parent, NetcdfDataset ds, Element netcdfElem, String uriString) throws IOException {
    ds.setUriString( uriString);
    ds.setId( netcdfElem.getAttributeValue("id"));

        // look for attributes
    java.util.List attList = netcdfElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      ucar.nc2.Attribute a = readAtt((Element) attList.get(j));
      ds.globalAttributes().add( a);
    }

    // look for dimensions
    java.util.List dimList = netcdfElem.getChildren("dimension", ncNS);
    for (int j=0; j< dimList.size(); j++) {
      Dimension d = readDim((Element) dimList.get(j));
      ds.dimensions().add( d);
    }

    // look for variables
    java.util.List varList = netcdfElem.getChildren("variable", ncNS);
    for (int j=0; j< varList.size(); j++) {
      Variable v = readVariable(ds, (Element) varList.get(j));
      ds.variables().add( v);
    }

    // process readMetadata command
    Element readElem = netcdfElem.getChild("readMetadata", ncNS);
    if (readElem != null) {
      readMetadata(ds, readElem);
    }

    /* look for coordinate variables
    java.util.List cvList = netcdfElem.getChildren("coordVariable", ncNS);
    for (int j=0; j< cvList.size(); j++) {
      Dimension d = readDim((Element) cvList.get(j));
      cmdRemove(ds, "dimension", d.getName()); // remove any copied from above
      ds.getDimensions().add( d);

      Variable v = readCoordVariable(ds, (Element) cvList.get(j));
      cmdRemove(ds, "variable", v.getName()); // remove any copied from above
      ds.getVariables().add( v);
    } */

   /*
    // copy parent objects into this dataset
    if (parent != null)
      addElementsCopy( parent, ds);
    // copy of Arraylist-dont want variables from nested datasets
    ArrayList myvars = new ArrayList( ds.variables());

    // read in nested netcdf elements
    java.util.List ncList = netcdfElem.getChildren("netcdf", ncNS);
    for (int j=0; j< ncList.size(); j++) {
      NetcdfDataset nestedDs = new NetcdfDataset();
      Element netcdfElemNested = (Element) ncList.get(j);
      String uriStringNested = netcdfElemNested.getAttributeValue("uri");
      readNetcdf(ds, nestedDs, netcdfElemNested, uriStringNested);
      ds.nestedDatasets().add( nestedDs);
      if (debugAgg) System.out.println("nested dataset uri= "+uriStringNested);
    }

    // copy child objects to parent
    // must have done all nested datasets before copying back to parent
    // note this is copy, not copyReplace
    ArrayList nestedDatasets = ds.nestedDatasets();
    for (int j=0; j< nestedDatasets.size(); j++) {
      NetcdfDataset nestedDs = (NetcdfDataset) nestedDatasets.get(j);
      addElements( nestedDs, ds);
    }

    // look for dimension values (joinNew aggregation)
    Element dvElem = netcdfElem.getChild("dimensionValue", ncNS);
    if (dvElem != null) {
      String dimName = dvElem.getAttributeValue("name");
      ds.setAggDimensionName(dimName);
      ds.setAggDimensionValue(dvElem.getAttributeValue("value"));

      // remove any dimension of this name
      Dimension aggDim = ds.findDimension( dimName);
      if (null != aggDim)
        ds.dimensions().remove( aggDim);

      // remove any variable dimension of this name
      // at this point, the dimensions are still encoded in the shapeS string
      ArrayList vars = ds.variables();
      for (int i=0; i<vars.size(); i++) {
        VariableDS v = (VariableDS) vars.get(i);
        v.removeDimension(dimName);
      }
    }

    // reset variable dimensions, derive shapes, etc
    // only on "myvars"
    for (int i=0; i<myvars.size(); i++) {
      VariableDS v = (VariableDS) myvars.get(i);
      v.finish();
    }

    // find any type1 and type2 aggregation variables
    convertAggNew(ds);
    convertAggExisting(ds);

    // last thing is to parse conventions : LOOK not sure about nested case
    Element pcElem = netcdfElem.getChild("parseConventions", ncNS);
    if (pcElem != null) {
      NetcdfDataset aug = ucar.nc2.dataset.conv.Convention.factory( ds);
      if (aug != null) ds = aug;
    }

    //System.out.println("End of factory parsing "+ds.getUrlString()+" ==\n"+ds+"\n");
  } */

  /* protected void convertAggNew(NetcdfDataset ds) {
    // find aggregation dimensions, if any
    ArrayList dims = ds.dimensions();
    for (int i=0; i<dims.size(); i++) {
      Dimension dim = (Dimension) dims.get(i);
      if (!dim.isUnlimited())   // must be unlimited dimension
        continue;

      int count = isAgg1Dimension(ds, dim);
      if (count > 0) {  // must have nested datasets
        dim.setLength( count);
        dim.setAggType( 1);

        // find variables that use this dimension
       ArrayList vars = ds.variables();
       for (int j=0; j<vars.size(); j++) {
          VariableDS v = (VariableDS) vars.get(j);
          List vdims = v.getDimensions();
          if (vdims.contains( dim)) { // actually must be first dimension
            Variable vagg = new VariableAggSynthetic( ds, v, dim);
            vars.set(j, vagg);
          }
        }

        // construct coordinate variable
        VariableDS coordVar = (VariableDS) ds.findVariable(dim.getName());
        if (coordVar == null) {
          coordVar = new VariableDS(ds, dim.getName(), "string", dim.getName(), null);
          coordVar.finish();
          vars.add( coordVar);
        }
        ArrayList coordValues = new ArrayList();
        ArrayList nested = ds.nestedDatasets();
        for (int k=0; k<nested.size(); k++) {
          NetcdfDataset nds = (NetcdfDataset) nested.get(k);
          // LOOK! assume only one type1 dimesnion !!
          coordValues.add( nds.getAggDimensionValue());
        }
        coordVar.setValues( coordValues);
      }
    }
  }

  protected int isAgg1Dimension(NetcdfDataset ds, Dimension dim) {
    if (dim.getAggType() != 0) return 0;
    int count = 0;

    // nested datasets must have dimensionValue element
    ArrayList nested = ds.nestedDatasets();
    for (int i=0; i<nested.size(); i++) {
      NetcdfDataset nds = (NetcdfDataset) nested.get(i);
      String dimName = nds.getAggDimensionName();
      if ((dimName != null) && dimName.equals(dim.getName()))
        count++;
    }

    return count;
  }

  protected void convertAggExisting(NetcdfDataset ds) {
    // find aggregation dimensions, if any
    ArrayList dims = ds.dimensions();
    for (int i=0; i<dims.size(); i++) {
      Dimension dim = (Dimension) dims.get(i);
      if (isAgg3Dimension( ds, dim)) {
        dim.setAggType( 3);

        // find aggregation variables
        ArrayList vars = ds.variables();
        for (int j=0; j<vars.size(); j++) {
          VariableDS v = (VariableDS) vars.get(j);
          if (isAgg3Variable( v, dim)) {
            if (debugAgg) System.out.println(" make VariableAggExisting "+v.getNameAndDimensions());
            VariableDS vagg = new VariableAggExisting( ds, (VariableDS) v);
            vars.set(j, vagg);
          }
        }
      }
    }
  }

  protected boolean isAgg3Dimension(NetcdfDataset ds, Dimension dim) {
    if (dim.getAggType() != 0) return false;

    int count = 0;
    int len = 0;

    // same dimension in nested dataset?
    ArrayList nested = ds.nestedDatasets();
    for (int i=0; i<nested.size(); i++) {
      NetcdfDataset nds = (NetcdfDataset) nested.get(i);
      Dimension nDim = nds.findDimension( dim.getName());
      if ((nDim != null) && (nDim != dim) && (nDim.getLength() < dim.getLength()))  {
        count++;
        len += nDim.getLength();
        // System.out.println("Dimension "+dim.getName()+"is agg type 3");
      }
    }
    // must have more than one match, lengths must add up
    if (count > 1) {
      if (len != dim.getLength())
        System.out.println("ERROR Aggregation Dimension "+dim.getName()+" len doesnt match");
      else
        return true;
    }

    return false;
  }

  protected boolean isAgg3Variable(VariableDS v, Dimension dim) {
    if (v.getRank() < 1) return false;
    if (v.isMetadata()) return false;

    Dimension outerDim = v.getDimension(0);
    return outerDim.equals( dim);
  } */

  ////////////////////////////////////////////////////////////////////////

  /**
   * Read an NcML attribute element.
   * @param parent Group or Variable
   * @param refParent Group or Variable in reference dataset
   * @param s ncml attribute element
   */
  protected void readAtt( Object parent, Object refParent, Element s) {
    String name = s.getAttributeValue("name");
    String nameInFile = s.getAttributeValue("orgName");
    if (nameInFile == null) nameInFile = name;

    // see if it already exists
    ucar.nc2.Attribute att = findAttribute( refParent, nameInFile);
    if (att == null) { // new
      if (debugConstruct) System.out.println(" add new att = "+name);
      ucar.ma2.Array values = readAttributeValues(s);
      if (values == null) return; // LOOK need error reporting
      addAttribute( parent, new ucar.nc2.Attribute( name, values));

    } else {
      att.setName( name);
      ucar.ma2.Array values = readAttributeValues(s);
      if (values != null)
        att.setValues( values);

      if (parent != refParent) // explicit
        addAttribute( parent, att);

      if (debugConstruct) System.out.println(" modify existing att = "+name);
    }
  }

  protected ucar.ma2.Array readAttributeValues(Element s) {
    String valString = s.getAttributeValue("value");
    if (valString == null) return null;
    valString = StringUtil.unquoteXmlAttribute(valString);

    String type = s.getAttributeValue("type");
    DataType dtype = (type == null) ? DataType.STRING : DataType.getType( type);
    if (dtype == DataType.CHAR) dtype = DataType.STRING;

    String sep = s.getAttributeValue("separator");
    if ((sep == null) && (dtype == DataType.STRING)) {
      ArrayList list = new ArrayList();
      list.add(valString);
      return NetcdfDataset.makeArray( dtype, list);
    }

    if (sep == null) sep = " "; // default whitespace separated

    ArrayList stringValues = new ArrayList();
    StringTokenizer tokn = new StringTokenizer(valString, sep);
    while (tokn.hasMoreTokens())
      stringValues.add( tokn.nextToken());

    return NetcdfDataset.makeArray( dtype, stringValues);
  }

  protected ucar.nc2.Attribute findAttribute( Object parent, String name) {
    if (parent == null)
      return null;
    if (parent instanceof Group)
      return ((Group) parent).findAttribute(name);
    else if (parent instanceof Variable)
      return ((Variable) parent).findAttribute(name);
    return null;
  }

  protected void addAttribute( Object parent, ucar.nc2.Attribute att) {
    if (parent instanceof Group)
      ((Group) parent).addAttribute(att);
    else if (parent instanceof Variable)
      ((Variable) parent).addAttribute(att);
  }

  /* protected VariableDS readCoordAxis( NetcdfDataset ds, Group g, Structure parentStructure, Element varElem) {
    VariableDS vds = readVariable( ds, g, parentStructure,  varElem);
    CoordinateAxis axis = CoordinateAxis.factory(ds, vds);

    String axisType = varElem.getAttributeValue("axisType");
    String positive = varElem.getAttributeValue("positive");
    String boundaryRef = varElem.getAttributeValue("boundaryRef");

    if (axisType != null)
      axis.setAxisType(AxisType.getType(axisType));
    if (positive != null)
      axis.setPositive(positive);
    if (boundaryRef != null)
      axis.setBoundaryRef(boundaryRef);

    return axis;
  }

  protected CoordinateSystem readCoordSystem( NetcdfDataset ds, Element csElem) {
    ArrayList axes = new ArrayList();
    ArrayList coordTrans = new ArrayList();

    // look for coordinate axis references
    java.util.List list = csElem.getChildren("coordinateAxisRef", ncNS);
    for (int j=0; j< list.size(); j++) {
     Element elem = (Element) list.get(j);
     String axisName = elem.getAttributeValue("ref");
     CoordinateAxis axis = ds.findCoordinateAxis(axisName);
     axes.add( axis);
     if (debug) System.out.println(" add coordinateAxisRef = "+axisName);
    }

     // look for coordinate transforms
    list = csElem.getChildren("coordinateTransform", ncNS);
    for (int j=0; j< list.size(); j++) {
     CoordinateTransform ct = readCoordTransform(ds, (Element) list.get(j));
     ds.addCoordinateTransform( ct);
     coordTrans.add( ct);
     if (debug) System.out.println(" add coordinateTransforms = "+ct.getName());
    }

    // look for coordinate transform references
    list = csElem.getChildren("coordinateTransformRef", ncNS);
    for (int j=0; j< list.size(); j++) {
     Element elem = (Element) list.get(j);
     String name = elem.getAttributeValue("ref");
     CoordinateTransform axis = ds.findCoordinateTransform(name);
     coordTrans.add( axis);
     if (debug) System.out.println(" add coordinateTransformRef = "+name);
    }

    return new CoordinateSystem(axes, coordTrans);
  }

  protected CoordinateTransform readCoordTransform( NetcdfDataset ds, Element ctElem) {
    String name = ctElem.getAttributeValue("name");
    String authority = ctElem.getAttributeValue("authority");
    String typeS = ctElem.getAttributeValue("type");

    TransformType type = (typeS == null) ? null : TransformType.getType( typeS);
    CoordinateTransform ct = new CoordinateTransform (name, authority, type);

    return ct;
  } */

  /**
   * Read the NcML dimension element.
   * @param refg parent Group in referenced dataset
   * @param s ncml dimension element
   */
  protected void readDim( Group g, Group refg, Element s) {
    String name = s.getAttributeValue("name");
    String nameInFile = s.getAttributeValue("orgName");
    if (nameInFile == null) nameInFile = name;

        // see if it already exists
    Dimension dim = refg.findDimension( nameInFile);
    if (dim == null) {
      String length = s.getAttributeValue("length");
      String isUnlimitedS = s.getAttributeValue("isUnlimited");
      String isSharedS = s.getAttributeValue("isShared");
      String isUnknownS = s.getAttributeValue("isUnknown");

      boolean isUnlimited = (isUnlimitedS != null) && isUnlimitedS.equalsIgnoreCase("true");
      boolean isUnknown = (isUnknownS != null) && isUnknownS.equalsIgnoreCase("true");
      boolean isShared = true;
      if ((isSharedS != null) && isSharedS.equalsIgnoreCase("false"))
        isShared = false;

      int len = Integer.parseInt( length);
      if ((isUnknownS != null) && isUnknownS.equalsIgnoreCase("false"))
        len = Dimension.UNKNOWN.getLength();

      if (debugConstruct) System.out.println(" add new dim = "+name);
      g.addDimension( new Dimension( name, len, isShared, isUnlimited, isUnknown));

    } else {

      String lengthS = s.getAttributeValue("length");
      String isUnlimitedS = s.getAttributeValue("isUnlimited");
      String isSharedS = s.getAttributeValue("isShared");
      String isUnknownS = s.getAttributeValue("isUnknown");

      if (isUnlimitedS != null)
        dim.setUnlimited( isUnlimitedS.equalsIgnoreCase("true"));

      if (isSharedS != null)
        dim.setShared( !isSharedS.equalsIgnoreCase("false"));

      if (isUnknownS != null)
        dim.setUnknown( isUnknownS.equalsIgnoreCase("true"));

      if ((lengthS != null) && !dim.isUnknown()) {
        int len = Integer.parseInt( lengthS);
        dim.setLength( len);
      }

      if (debugConstruct) System.out.println(" modify existing dim = "+name);

      if (g != refg) // explicit
        g.addDimension( dim);
    }
  }

  /**
   * Read the NcML group element, and nested elements.
   * @param newds new dataset
   * @param refds referenced dataset
   * @param parent Group
   * @param refParent parent Group in referenced dataset
   * @param groupElem ncml group element
   */
  protected void readGroup( NetcdfDataset newds, NetcdfDataset refds, Group parent, Group refParent, Element groupElem)  {
    String name = groupElem.getAttributeValue("name");
    String nameInFile = groupElem.getAttributeValue("orgName");
    if (nameInFile == null) nameInFile = name;

    Group g, refg;
    if (parent == null) {
      g = newds.getRootGroup();
      refg = refds.getRootGroup();
      if (debugConstruct) System.out.println(" root group ");

    } else {
      // see if it exists in referenced dataset
      refg = refParent.findGroup( nameInFile);
      if (refg == null) { // new
        g =  new Group( newds, parent, name);
        parent.addGroup( g);
        if (debugConstruct) System.out.println(" add new group = "+name);

      } else {

        if (parent != refParent) { // explicit
          g =  new Group( newds, parent, name);
          parent.addGroup( g);
          if (debugConstruct) System.out.println(" transfer existing group = "+name);

        } else { // modify
          g = refg;
          if (!nameInFile.equals(name))
            g.setName( name); // LOOK this renames all its variables !!

          if (debugConstruct) System.out.println(" modify existing group = "+name);
        }
      }
    }

        // look for attributes
    java.util.List attList = groupElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      readAtt( g, refg, (Element) attList.get(j));
    }

    // look for dimensions
    java.util.List dimList = groupElem.getChildren("dimension", ncNS);
    for (int j=0; j< dimList.size(); j++) {
      readDim(g, refg, (Element) dimList.get(j));
     }

    // look for variables
    java.util.List varList = groupElem.getChildren("variable", ncNS);
    for (int j=0; j< varList.size(); j++) {
      readVariable(newds, g, refg, (Element) varList.get(j));
    }

    // process remove command
    java.util.List removeList = groupElem.getChildren("remove", ncNS);
    for (int j=0; j< removeList.size(); j++) {
      Element e = (Element) removeList.get(j);
      cmdRemove(g, e.getAttributeValue("type"), e.getAttributeValue("name"));
    }

    // look for nested groups
    java.util.List groupList = groupElem.getChildren("group", ncNS);
    for (int j=0; j< groupList.size(); j++) {
      readGroup(newds, refds, g, refg, (Element) groupList.get(j));
      if (debugConstruct) System.out.println(" add group = "+g.getName());
    }
  }

  /* private boolean debugView = false, debugConvert = false;
  protected VariableDS readVariable2( NetcdfDataset ds, Element varElem) {
    VariableDS v = readVariable( ds, varElem);

    // look for logical views
    java.util.List viewList = varElem.getChildren("logicalView", ncNS);
    for (int j=0; j< viewList.size(); j++) {
      Element viewElem = (Element) viewList.get(j);
      String value = viewElem.getAttributeValue("section");
      if (value != null) {
        v.setLogicalView("section", value);
        if (debugView) System.out.println("set view = "+value);
      }
    }

    // look for unit conversion
    Element unitElem = varElem.getChild("units", ncNS);
    if (unitElem != null) {
      String value = unitElem.getAttributeValue("convertTo");
      if (value != null) {
        v.setConvertUnit(value);
        if (debugConvert) System.out.println("setConvertUnit on "+v.getName()+" to <" + value+">");
      }
    }

    return v;
     } */

  /**
   * Read the NcML variable element, and nested elements.
   * @param g parent Group
   * @param refg referenced dataset parent Group - may be same (modify) or different (explicit)
   * @param varElem ncml variable element
   */
  protected void readVariable( NetcdfDataset ds, Group g, Group refg, Element varElem) {
    String name = varElem.getAttributeValue("name");
    String nameInFile = varElem.getAttributeValue("orgName");
    if (nameInFile == null) nameInFile = name;

    // see if it already exists
    Variable refv = refg.findVariable( nameInFile);
    if (refv == null) { // new
      if (debugConstruct) System.out.println(" add new var = "+name);
      g.addVariable( readVariableNew(ds, g, null, varElem));
      return;
    }

    // exists already
    DataType dtype = null;
    String typeS = varElem.getAttributeValue("type");
    if (typeS != null)
      dtype = DataType.getType(typeS);
    else
      dtype = refv.getDataType();

    String shape = varElem.getAttributeValue("shape");
    if (shape == null) shape = refv.getDimensionsString();

    Variable v;
    if (refg == g) { // modify
      v = refv;
      v.setName( name);
      v.setDataType( dtype);
      v.setDimensions( shape);
      if (debugConstruct) System.out.println(" modify existing var = "+nameInFile);

    } else { //explicit
      if (refv instanceof Structure) {
        v = new StructureDS(ds, g, null, name, shape, null, null);
      } else {
        v = new VariableDS(ds, g, null, name, dtype, shape, null, null);
      }
      v.setIOVar( refv);
      if (debugConstruct) System.out.println(" modify explicit var = "+nameInFile);
      g.addVariable( v);
    }

    java.util.List attList = varElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      readAtt(v, refv, (Element) attList.get(j));
    }

     // process remove command
    java.util.List removeList = varElem.getChildren("remove", ncNS);
    for (int j=0; j< removeList.size(); j++) {
      Element e = (Element) removeList.get(j);
      cmdRemove(v, e.getAttributeValue("type"), e.getAttributeValue("name"));
    }

    if (v.getDataType() == DataType.STRUCTURE) {
      // deal with nested variables
      StructureDS s = (StructureDS) v;
      StructureDS refS = (StructureDS) refv;
      java.util.List varList = varElem.getChildren("variable", ncNS);
      for (int j=0; j< varList.size(); j++) {
        readVariableNested(ds, s, refS, (Element) varList.get(j));
      }

    } else {

          // deal with values
      Element valueElem = varElem.getChild("values", ncNS);
      if (valueElem != null)
        readValues( ds, v, varElem, valueElem);
    }

  }

  /**
   * Read a NcML variable element, and nested elements, when it creates a new Variable.
   * @param g parent Group
   * @param parentS parent Structure
   * @param varElem ncml variable element
   * @return return new Variable
   */
  protected Variable readVariableNew( NetcdfDataset ds, Group g, Structure parentS, Element varElem) {
    String name = varElem.getAttributeValue("name");
    String type = varElem.getAttributeValue("type");
    String shape = varElem.getAttributeValue("shape");

    DataType dtype = DataType.getType(type);

    Variable v;

    if (dtype == DataType.STRUCTURE) {
      StructureDS s = new StructureDS( ds, g, parentS, name, shape, null, null);
      v = s;
       // look for nested variables
      java.util.List varList = varElem.getChildren("variable", ncNS);
      for (int j=0; j< varList.size(); j++) {
        readVariableNested(ds, s, s, (Element) varList.get(j));
      }

    } else {
      v = new VariableDS( ds, g, parentS, name, dtype, shape, null, null);
      Element valueElem = varElem.getChild("values", ncNS);
          // deal with values
      if (valueElem != null)
        readValues( ds, v, varElem, valueElem);
    }

    // look for attributes
    java.util.List attList = varElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      readAtt(v, null, (Element) attList.get(j));
    }

    return v;
  }

 /**
  * Read the NcML variable element, and nested elements.
  * @param parentS parent Structure
  * @param varElem ncml variable element
  */
  protected void readVariableNested( NetcdfDataset ds, Structure parentS, Structure refStruct, Element varElem) {
   String name = varElem.getAttributeValue("name");
   String nameInFile = varElem.getAttributeValue("orgName");
   if (nameInFile == null) nameInFile = name;

   // see if it already exists
   Variable refv = refStruct.findVariable( nameInFile);
   if (refv == null) { // new
     if (debugConstruct) System.out.println(" add new var = "+name);
     Variable nested = readVariableNew(ds, parentS.getParentGroup(), parentS, varElem);
     parentS.addMemberVariable( nested);
     return;
   }

   Variable v;
   if (parentS == refStruct) { // modify
     v = refv;
     v.setName( name);

   } else { //explicit
     if (refv instanceof Structure) {
       v = new StructureDS(ds, parentS.getParentGroup(), parentS, name, refv.getDimensionsString(), null, null);
     } else {
       v = new VariableDS(ds, parentS.getParentGroup(), parentS, name, refv.getDataType(), refv.getDimensionsString(), null, null);
     }
     v.setIOVar( refv);
     parentS.addMemberVariable( v);
   }

   if (debugConstruct) System.out.println(" modify existing var = "+nameInFile);

   String typeS = varElem.getAttributeValue("type");
   if (typeS != null) {
     DataType dtype = DataType.getType(typeS);
     v.setDataType( dtype);
   }

   String shape = varElem.getAttributeValue("shape");
   if (shape != null) {
     v.setDimensions( shape);
   }

   java.util.List attList = varElem.getChildren("attribute", ncNS);
   for (int j=0; j< attList.size(); j++) {
     readAtt(v, refv, (Element) attList.get(j));
   }

    // process remove command
   java.util.List removeList = varElem.getChildren("remove", ncNS);
   for (int j=0; j< removeList.size(); j++) {
     Element e = (Element) removeList.get(j);
     cmdRemove(v, e.getAttributeValue("type"), e.getAttributeValue("name"));
   }

   if (v.getDataType() == DataType.STRUCTURE) {
     // deal with nested variables
     StructureDS s = (StructureDS) v;
     StructureDS refS = (StructureDS) refv;
     java.util.List varList = varElem.getChildren("variable", ncNS);
     for (int j=0; j< varList.size(); j++) {
       readVariableNested(ds, s, refS, (Element) varList.get(j));
     }

   } else {

         // deal with values
     Element valueElem = varElem.getChild("values", ncNS);
     if (valueElem != null)
       readValues( ds, v, varElem, valueElem);
    }
  }

  protected void readValues( NetcdfDataset ds, Variable v, Element varElem, Element valuesElem) {
    ArrayList valList = new ArrayList();

    // should really be a seperate element
    String startS = valuesElem.getAttributeValue("start");
    String incrS = valuesElem.getAttributeValue("increment");
    String nptsS = valuesElem.getAttributeValue("npts");

    // either start, increment are specified
    if ((startS != null) && (incrS != null)) {
      double start = Double.parseDouble( startS);
      double incr = Double.parseDouble( incrS);
      int npts = Integer.parseInt( nptsS);

      ds.setValues( v, npts, start, incr);
      return;
    }

    String values = varElem.getChildText("values", ncNS);
    String sep = varElem.getAttributeValue("separator");
    if (sep == null) sep = " ";

    if (v.getDataType() == DataType.CHAR) {
      int nhave = values.length();
      int nwant = (int) v.getSize();
      char[] data = new char[nwant];
      int min = Math.min(nhave, nwant);
      for (int i=0; i<min; i++) {
         data[i] = values.charAt(i);
      }
      Array dataArray = Array.factory( DataType.CHAR.getPrimitiveClassType(), v.getShape(), data);
      v.setCachedData( dataArray, true);

    } else {
      // or a list of values
      StringTokenizer tokn = new StringTokenizer(values, sep);
      while (tokn.hasMoreTokens())
        valList.add(tokn.nextToken());
      ds.setValues( v, valList);
    }
  }

  /////////////////////////////////////////////////////////////////////////////////////////
  private Aggregation readAgg( Element aggElem, CancelTask cancelTask) throws IOException {
    String dimName = aggElem.getAttributeValue("dimName");
    String type = aggElem.getAttributeValue("type");
    Aggregation agg = new Aggregation( this, dimName, type);

    // look for variable names
    java.util.List list = aggElem.getChildren("variableAgg", ncNS);
    for (int j=0; j< list.size(); j++) {
      Element e = (Element) list.get(j);
      String varName = e.getAttributeValue("name");
      agg.vars.add( varName);
    }

       // nested netcdf elements
    java.util.List ncList = aggElem.getChildren("netcdf", ncNS);
    for (int j=0; j< ncList.size(); j++) {
      Element netcdfElemNested = (Element) ncList.get(j);
      String location = netcdfElemNested.getAttributeValue("location");
      if (location == null)
        location = netcdfElemNested.getAttributeValue("uri");
      String ncoords = netcdfElemNested.getAttributeValue("ncoords");
      String coordValueS = netcdfElemNested.getAttributeValue("coordValue");

      agg.addDataset( location, ncoords, coordValueS, netcdfElemNested, cancelTask);
      if ((cancelTask != null) && cancelTask.isCancel())
        return null;
      if (debugAggDetail) System.out.println(" debugAgg: nested dataset = "+location);
    }

    return agg;
  }

  private void aggExistingDimension( NetcdfDataset newds, Aggregation agg, CancelTask cancelTask) throws IOException {
    // create aggregation dimension
    Dimension aggDim = new Dimension( agg.dimName, agg.getNcoords(), true);
    newds.removeDimension( null, agg.dimName); // remove previous declaration, if any
    newds.addDimension( null, aggDim);
    if (debugAgg) System.out.println(" add aggExisting dimension = "+aggDim);

    // always open the first nested dataset and copy it to newds
    Aggregation.Dataset firstNested = (Aggregation.Dataset) agg.nestedDatasets.get(0);
    NetcdfDataset firstDs = firstNested.openDataset( cancelTask);
    transferDataset(firstNested.ds, newds);

    // now we can create the real aggExisting variables
    // all variables with the named aggregation dimension
    List vars = firstDs.getVariables();
    for (int i=0; i<vars.size(); i++) {
      Variable v = (Variable) vars.get(i);
      Dimension d = v.getDimension(0);
      if (!agg.dimName.equals(d.getName()))
        continue;

      VariableDS vagg = new VariableDS(newds, null, null, v.getShortName(), v.getDataType(),
          v.getDimensionsString(), null, null);
      vagg.setAggregation( agg);
      transferVariableAttributes( v, vagg);

      newds.removeVariable( null, v.getShortName());
      newds.addVariable( null, vagg);
    }
  }

  private void aggNewDimension( NetcdfDataset newds, Aggregation agg, CancelTask cancelTask) throws IOException {
    // create aggregation dimension
    Dimension aggDim = new Dimension( agg.dimName, agg.getNcoords(), true);
    newds.removeDimension( null, agg.dimName); // remove previous declaration, if any
    newds.addDimension( null, aggDim);
    if (debugAgg) System.out.println(" add aggNew dimension = "+aggDim);

    // create aggregation coordinate variable
    DataType coordType = null;
    Variable coordVar = newds.getRootGroup().findVariable(agg.dimName);
    if (coordVar == null) {
      coordType = agg.getCoordinateType();
      coordVar = new VariableDS(newds, null, null, agg.dimName, coordType, agg.dimName, null, null);
      newds.addVariable(null, coordVar);
    } else {
      coordType = coordVar.getDataType();
      coordVar.setDimensions(agg.dimName); // reset its dimension
    }

    // set its values
    int[] shape = new int[] { agg.getNcoords() };
    Array coordData = Array.factory(coordType.getPrimitiveClassType(), shape);
    Index ima = coordData.getIndex();
    List nestedDataset = agg.getNestedDatasets();
    for (int i = 0; i < nestedDataset.size(); i++) {
      Aggregation.Dataset nested = (Aggregation.Dataset) nestedDataset.get(i);
      if (coordType == DataType.STRING)
        coordData.setObject(ima.set(i), nested.coordValueS);
      else
        coordData.setDouble(ima.set(i), nested.coordValue);
    }
    coordVar.setCachedData( coordData, true);

    // always open the first nested dataset and copy it to newds
    Aggregation.Dataset firstNested = (Aggregation.Dataset) agg.nestedDatasets.get(0);
    NetcdfDataset firstDs = firstNested.openDataset( cancelTask);
    transferDataset(firstNested.ds, newds);

    // now we can create all the aggNew variables
    // use only named variables
    List vars = agg.vars;
    for (int i=0; i<vars.size(); i++) {
      String varname = (String) vars.get(i);
      Variable v = newds.getRootGroup().findVariable(varname);
      if (v == null) {
        System.out.println("aggNewDimension cant find variable "+varname);
        continue;
      }

      // construct new variable, replace old one
      VariableDS vagg = new VariableDS(newds, null, null, v.getShortName(), v.getDataType(),
          agg.dimName +" "+ v.getDimensionsString(), null, null);
      vagg.setAggregation( agg);
      transferVariableAttributes( v, vagg);

      newds.removeVariable( null, v.getShortName());
      newds.addVariable( null, vagg);
    }
  }


  /* protected Variable readCoordVariable( NetcdfDataset ds, Element varElem) {
    String name = varElem.getAttributeValue("name");
    String type = varElem.getAttributeValue("type");
    String shape;
    if (type.equals("string"))
      shape = name+" "+name+"_len";
    else
      shape = name;

    VariableDS v = new VariableDS( ds, name, type, shape, null);

    // look for attributes
    java.util.List attList = varElem.getChildren("attribute", ncNS);
    for (int j=0; j< attList.size(); j++) {
      Attribute a = readAtt((Element) attList.get(j));
      v.addAttribute( a);
    }

    // look for values
    String values = varElem.getAttributeValue("value");
    if (values == null)
      values = varElem.getChildText("values", ncNS);
    if (values != null) {
      ArrayList valList = new ArrayList();
      String sep = varElem.getAttributeValue("separator");
      if (sep == null) sep = " ";
      StringTokenizer tokn = new StringTokenizer(values, sep);
      while (tokn.hasMoreTokens())
        valList.add(tokn.nextToken());
      v.setValues( valList);
    }

    return v;
  } */

  /* protected void readMetadata( NetcdfDataset ds, Element readElem, CancelTask cancelTask) throws IOException {
    //String name = readElem.getAttributeValue("name"); // not implemented yet
    //String type = readElem.getAttributeValue("type"); // not implemented yet

    // read dataset
    NetcdfDataset referencedDataset = ds.openReferencedDataset(cancelTask);
    transferDataset( referencedDataset, ds);
    if (debugCmd) System.out.println("CMD readMetadata "+referencedDataset.getLocation());

    /* remove elements just read
    java.util.List removeList = readElem.getChildren("remove", ncNS);
    for (int j=0; j< removeList.size(); j++) {
      Element e = (Element) removeList.get(j);
      cmdRemove(ds, e.getAttributeValue("type"), e.getAttributeValue("name"));
    }

    // look for renames
    java.util.List renameList = readElem.getChildren("rename", ncNS);
    for (int j=0; j< renameList.size(); j++) {
      Element e = (Element) renameList.get(j);
      cmdRename(ds, e.getAttributeValue("type"), e.getAttributeValue("nameInFile"),
        e.getAttributeValue("rename"));
    }

    java.util.List varCmdList = readElem.getChildren("readVariable", ncNS);
    for (int j=0; j< varCmdList.size(); j++) {
      Element e = (Element) varCmdList.get(j);
      readReadVariable( ds, e);
    }

  } */

  /* protected void readReadVariable( NetcdfDataset ds, Element readVarElem) {
      String varName = readVarElem.getAttributeValue("name");

      String rename = readVarElem.getAttributeValue("rename");
      if (rename != null) {
        cmdRename( ds, "variable", varName, rename);
        varName = rename;
      }

      VariableDS v = (VariableDS) ds.findVariable(varName);
      if (v == null) {
        System.out.println("NetcdfDataset (readReadVariable) cant find variable= "+varName);
        return;
      }

      java.util.List removeList = readVarElem.getChildren("removeAttribute", ncNS);
      for (int j=0; j< removeList.size(); j++) {
        Element e = (Element) removeList.get(j);
        cmdRemoveVarAtt(ds, v, e.getAttributeValue("name"));
      }

      java.util.List renameList = readVarElem.getChildren("renameAttribute", ncNS);
      for (int j=0; j< renameList.size(); j++) {
        Element e = (Element) renameList.get(j);
        cmdRenameVarAtt(ds, v, e.getAttributeValue("nameInFile"), e.getAttributeValue("rename"));
      }

      java.util.List addList = readVarElem.getChildren("attribute", ncNS);
      for (int j=0; j< addList.size(); j++) {
        Element e = (Element) addList.get(j);
        cmdAddVarAtt(ds, v, e);
      }
    } */

  /////////////////////////////////////////////
  // command procesing

  /**
   * Copy contents of "from" to "to", as long as "to" doesnt already have
   * an elements of that name.
   *
  protected void addElements(NetcdfFile from, NetcdfDataset ds) {

    // transfer everything from existing file into this dataset
    // that doesnt already exist
    Iterator iterDim = from.getDimensions().iterator();
    while (iterDim.hasNext()) {
      Dimension d = (Dimension) iterDim.next();
      if (!ds.dimensions().contains(d)) {
        ds.dimensions().add( d);
      }
    }

    Iterator iterVar = from.getVariables().iterator();
    while (iterVar.hasNext()) {
      Variable v = (Variable) iterVar.next();
      if (!ds.variables().contains(v)) {
        ds.variables().add( v);
        if (debugAgg) System.out.println(" copy var "+v.getName());
      }
    }

    Iterator iterAtt = from.getGlobalAttributes().iterator();
    while (iterAtt.hasNext()) {
      ucar.nc2.Attribute a = (ucar.nc2.Attribute) iterAtt.next();
      if (!ds.globalAttributes().contains(a))
        ds.globalAttributes().add( a);
    }
  }

  /**
   * Copy contents of "src" to "target", as long as "ds" doesnt already have
   * an element of that name. Dimensions and variables are replaced with equivalent
   * elements. Attribute doesnt have to be replaced because its immutable.
   */
  protected void transferDataset(NetcdfDataset src, NetcdfDataset target) {
    transferGroup( src, src.getRootGroup(), target.getRootGroup());
  }

  private void transferGroup( NetcdfDataset ds, Group src, Group target) {

    Iterator iterAtt = src.getAttributes().iterator();
    while (iterAtt.hasNext()) {
      ucar.nc2.Attribute a = (ucar.nc2.Attribute) iterAtt.next();
      if (null == target.findAttribute( a.getName()))
        target.addAttribute(a);
    }

    Iterator iterDim = src.getDimensions().iterator();
    while (iterDim.hasNext()) {
      Dimension d = (Dimension) iterDim.next();
      if (null == target.findDimensionLocal(d.getName())) {
        // gets its own copy of Dimension
        Dimension newd = new Dimension(d.getName(), d.getLength(), d.isShared(), d.isUnlimited(), d.isUnknown());
        target.addDimension( newd);
      }
    }

    Iterator iterVar = src.getVariables().iterator();
    while (iterVar.hasNext()) {
      Variable v = (Variable) iterVar.next();
      if (null == target.findVariable(v.getShortName())) {
        v.setDimensions( v.getDimensionsString()); // LOOK rediscover dimensions
        target.addVariable( v);
      }
    }

    for (Iterator iter = src.getGroups().iterator(); iter.hasNext(); ) {
      Group srcNested = (Group) iter.next();
      Group nested = new Group( ds, target, srcNested.getName());
      target.addGroup( nested);
      transferGroup( ds, srcNested, nested);
    }
  }

  private void transferVariableAttributes( Variable src, Variable target) {
    Iterator iterAtt = src.getAttributes().iterator();
    while (iterAtt.hasNext()) {
      ucar.nc2.Attribute a = (ucar.nc2.Attribute) iterAtt.next();
      if (null == target.findAttribute( a.getName()))
        target.addAttribute(a);
    }
  }

  protected void cmdRemove(Group g, String type, String name) {
    if (type.equals("dimension"))  {
      Dimension dim = g.findDimension(name);
      if (dim != null) {
        g.remove( dim);
        if (debugCmd) System.out.println("CMD remove "+type+" "+name);
      }
    } else if (type.equals("variable"))  {
      Variable v = g.findVariable(name);
      if (v != null) {
        g.remove( v);
        if (debugCmd) System.out.println("CMD remove "+type+" "+name);
      }
    } else if (type.equals("attribute"))  {
      ucar.nc2.Attribute a = g.findAttribute(name);
      if (a != null) {
        g.remove( a);
        if (debugCmd) System.out.println("CMD remove "+type+" "+name);
      }
    }
  }

  protected void cmdRemove(Variable v, String type, String name) {
   if (type.equals("attribute"))  {
      ucar.nc2.Attribute a = v.findAttribute(name);
      if (a != null) {
        v.remove( a);
        if (debugCmd) System.out.println("CMD remove "+type+" "+name);
      }
    } else if (type.equals("variable") && v instanceof Structure)  {
      Structure s = (Structure) v;
      Variable nested = s.findVariable(name);
      if (nested != null) {
        s.removeMemberVariable( nested);
        if (debugCmd) System.out.println("CMD remove "+type+" "+name);
      }
    }
  }

  /* protected void cmdRename(NetcdfDataset ds, String type, String name, String rename) {
    if (type.equals("variable")) {
      VariableDS v = (VariableDS) ds.findVariable(name);
      if (v == null) {
        System.out.println("NetcdfDataset cmdRename cant find variable= "+name);
        return;
      }
      v.rename( rename);
      v.setAlias( name);
      if (debugCmd) System.out.println("CMD rename var "+name+" to "+rename);

    } else if (type.equals("attribute")) {
      ucar.nc2.Attribute oldAtt = ds.findGlobalAttribute( name);
      if (null == oldAtt) {
        System.out.println("NetcdfDataset cmdRename cant find global attribute "+name);
        return;
      }

      oldAtt.rename( rename);
      if (debugCmd) System.out.println("CMD rename att "+name+" renamed to "+rename);

    } else if (type.equals("dimension")) {
      ucar.nc2.Dimension d = ds.findDimension( name);
      if (null == d) {
        System.out.println("NetcdfDataset cmdRename cant find dimension "+name);
        return;
      }
      d.rename( rename);
      if (debugCmd) System.out.println("CMD rename dim "+name+" renamed to "+rename);
    }
  }

  protected void cmdAddVarAtt(NetcdfDataset ds, VariableDS v, Element e) {
    ucar.nc2.Attribute newAtt = readAtt(e);
    ucar.nc2.Attribute oldAtt = v.findAttribute( newAtt.getName());
    if (null != oldAtt)
      v.removeAttribute( oldAtt);
    ds.addVariableAttribute( v, newAtt);
    if (debugCmd) System.out.println("CMD variable added att "+newAtt);
  }

  protected void cmdRemoveVarAtt(NetcdfDataset ds, VariableDS v, String attName) {
    ucar.nc2.Attribute oldAtt = v.findAttribute( attName);
    if (null != oldAtt) {
      v.removeAttribute( oldAtt);
      if (debugCmd) System.out.println("CMD variable removed att "+attName);
    } else
      System.out.println("NetcdfDataset cant find attribute "+attName+" variable "+v.getName());
  }

  protected void cmdRenameVarAtt(NetcdfDataset ds, Variable v, String attName, String newName) {
    ucar.nc2.Attribute oldAtt = v.findAttribute( attName);
    if (null != oldAtt) {
      oldAtt.rename( newName);
      if (debugCmd) System.out.println("CMD variable att "+attName+" renamed to "+newName);
    } else
      System.out.println("NetcdfDataset cmdRenameVarAtt cant find attribute "+attName+
        " variable "+v.getName());
  } */

  public static void main( String arg[]) {
    String urls = "file:C:/dev/netcdf-java-2.2/test/data/dataset/xml/tmp/ocean.nc.ncml";
    try {
      NetcdfDataset ds = new NcMLReader().readNcML(urls, null);
      System.out.println("NetcdfDataset = "+urls+"\n"+ds);
    } catch (Exception ioe) {
      System.out.println("error = "+urls);
      ioe.printStackTrace();
    }
  }

}

/* Change History:
   $Log: NcMLReader.java,v $
   Revision 1.5  2004/12/10 19:44:48  caron
   no message

   Revision 1.4  2004/12/07 01:29:31  caron
   redo convention parsing, use _Coordinate encoding.

   Revision 1.3  2004/12/03 04:46:26  caron
   no message

   Revision 1.2  2004/12/01 05:53:42  caron
   ncml pass 2, new convention parsing

   Revision 1.1  2004/11/21 01:16:47  caron
   ncml pass 1

   Revision 1.9  2004/11/17 19:53:08  caron
   no message

   Revision 1.8  2004/11/16 23:35:38  caron
   no message

   Revision 1.7  2004/11/07 03:00:49  caron
   *** empty log message ***

   Revision 1.6  2004/11/04 00:38:18  caron
   no message

   Revision 1.5  2004/10/29 00:14:10  caron
   no message

   Revision 1.4  2004/10/06 19:03:41  caron
   clean up javadoc
   change useV3 -> useRecordsAsStructure
   remove id, title, from NetcdfFile constructors
   add "in memory" NetcdfFile

   Revision 1.3  2004/09/22 18:43:02  caron
   move common to ucar.unidata; projections use Parameter, no tAttribute

   Revision 1.2  2004/08/26 17:55:07  caron
   no message

   Revision 1.1  2004/08/16 20:53:48  caron
   2.2 alpha (2)

   Revision 1.6  2003/09/22 22:28:04  caron
   fix NcML agg errors: interval calculation and selfContained variables shouldnt be agg'd

   Revision 1.5  2003/09/19 00:11:41  caron
   debugging and javadoc fixes

   Revision 1.4  2003/09/02 22:27:59  caron
   NcML dataset version 2

   Revision 1.3  2003/06/19 22:58:07  caron
   bug in nested elements

   Revision 1.2  2003/05/29 23:45:19  caron
   add Constructor that allows uri substitution

   Revision 1.1  2003/04/08 15:06:24  caron
   nc2 version 2.1

 */

  /* public void writeXML(InvCatalogImpl catalog, OutputStream os) throws IOException {
    // Output the document, use standard formatter
    XMLOutputter fmt = new XMLOutputter("  ", true);
    fmt.output(makeCatalog(catalog), os);
  } */



