// $Id: DODSNetcdfFile.java,v 1.25 2004/12/10 17:04:17 caron Exp $
/*
 * Copyright 1997-2004 Unidata Program Center/University Corporation for
 * Atmospheric Research, P.O. Box 3000, Boulder, CO 80307,
 * support@unidata.ucar.edu.
 *
 * This library is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or (at
 * your option) any later version.
 *
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser
 * General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this library; if not, write to the Free Software Foundation,
 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package ucar.nc2.dods;

import ucar.ma2.*;
import ucar.nc2.*;
import ucar.nc2.util.*;
import ucar.unidata.util.StringUtil;
import dods.dap.*;

import java.io.*;
import java.util.*;
import dods.dap.parser.*;

/**
 * Access to DODS datasets through the Netcdf API.
 *
 * @see ucar.nc2.NetcdfFile
 * @author caron
 * @version $Revision: 1.25 $ $Date: 2004/12/10 17:04:17 $
 */

public class DODSNetcdfFile extends ucar.nc2.NetcdfFile {
  static public boolean debugCE = false;
  static public boolean debugServerCall = false;
  static public boolean debugOpenResult = false;
  static public boolean debugDataResult = false;
  static public boolean debugCharArray = false;
  static public boolean debugConvertData = false;
  static public boolean debugConstruct = false;
  static public boolean debugTime = false;
  static public boolean showNCfile = false;
  static public boolean debugAttributes = false;
  static public boolean debugCached = false;

  static public void setDebugFlags( ucar.nc2.util.DebugFlags debugFlag) {
    debugCE =  debugFlag.isSet("DODS/constraintExpression");
    debugServerCall =  debugFlag.isSet("DODS/serverCall");
    debugOpenResult =  debugFlag.isSet("DODS/debugOpenResult");
    debugDataResult =  debugFlag.isSet("DODS/debugDataResult");
    debugCharArray =  debugFlag.isSet("DODS/charArray");
    debugConstruct =  debugFlag.isSet("DODS/constructNetcdf");
    debugTime =  debugFlag.isSet("DODS/timeCalls");
    showNCfile =  debugFlag.isSet("DODS/showNCfile");
    debugAttributes =  debugFlag.isSet("DODS/attributes");
    debugCached =  debugFlag.isSet("DODS/cache");
  }

  static private boolean preload = true;
  static private boolean useGroups = false;
  /**
   * Set whether small variables are preloaded; only turn off for debugging.
   */
  static void setPreload( boolean b) { preload = b; }
  /**
   * Create the canonical form of the URL.
   * If the urlName starts with "http:", change it to start with "dods:", otherwise
   * leave it alone.
   * @param urlName the url string
   * @return canonical form
   */
  public static String canonicalURL( String urlName) {
    if (urlName.startsWith("http:"))
      return"dods:" + urlName.substring(5);
    return urlName;
  }

  //////////////////////////////////////////////////////////////////////////////////
  private boolean accept_deflate = false; // ??
  private ArrayList ddsVlist; // tree of DodsV objects, corresponding to DDS
  private DConnect dodsConnection = null;
  private DDS ddsSave; // for debugging
  private DAS dasSave;
  private ServerVersion dodsVersion;

   /**
    * Open a DODS file.
    * @param datasetURL URL of the file. This should start with the protocol "dods:"
    *   It may also start with protocol "http:".
    * @throws IOException
    * @throws DODSException
    * @throws java.net.MalformedURLException
    */
  public DODSNetcdfFile(String datasetURL) throws IOException, DODSException, java.net.MalformedURLException {
    this(datasetURL, null);
  }

   /**
    * Open a DODS file, allow user control over preloading string arrays and making structure data
    *   available through netcdf API.
    * @param datasetURL URL of the file. This should start with the protocol "dods:" or "http:".
    * @param cancelTask check if task is cancelled. may be null.
    * @throws IOException
    * @throws DODSException
    * @throws java.net.MalformedURLException
    */
  public DODSNetcdfFile(String datasetURL, CancelTask cancelTask)
      throws IOException, DODSException, java.net.MalformedURLException {

    super();

    // canonicalize name
    String urlName = datasetURL; // actual URL uses http:
    this.location = datasetURL; // canonical name uses "dods:"
    if (datasetURL.startsWith("dods:")) {
      urlName = "http:" + datasetURL.substring(5);
    } else if (datasetURL.startsWith("http:")) {
      this.location = "dods:" + datasetURL.substring(5);
    } else {
      throw new java.net.MalformedURLException(datasetURL+" must start with dods: or http:");
    }

    if (debugServerCall) System.out.println("DConnect to = <"+urlName+">");
    dodsConnection = new DConnect(urlName, accept_deflate);
    if (cancelTask != null && cancelTask.isCancel()) return;

    // fetch the DDS and DAS
    DDS dds = null;
    DAS das = null;
    try {
      dds = dodsConnection.getDDS();
      if (debugServerCall) System.out.println("DODSNetcdfFile readDDS");
      if (debugOpenResult) {
        System.out.println("DDS = ");
        dds.print(System.out);
      }
      if (cancelTask != null && cancelTask.isCancel()) return;

      das = dodsConnection.getDAS();
      if (debugServerCall) System.out.println("DODSNetcdfFile readDAS");
      if (debugOpenResult) {
        System.out.println("DAS = ");
        das.print(System.out);
      }
      if (cancelTask != null && cancelTask.isCancel()) return;

      dodsVersion = dodsConnection.getServerVersion();
      if (debugOpenResult) System.out.println("dodsVersion = "+dodsVersion);
    } catch (dods.dap.parser.ParseException e) {
      System.out.println("ERROR = "+e.getMessage());
      throw new IOException( e.getMessage());
    }

    // now initialize the DODSNetcdf metadata
    ddsVlist = parseDDS(dds);
    parseDAS(das, ddsVlist);
    if (cancelTask != null && cancelTask.isCancel()) return;

    constructTopVariables(ddsVlist, cancelTask);
    if (cancelTask != null && cancelTask.isCancel()) return;

    //preload(dodsVlist, cancelTask); LOOK not using preload yet
    //if (cancelTask != null && cancelTask.isCancel()) return;

    constructConstructors(ddsVlist, cancelTask);
    if (cancelTask != null && cancelTask.isCancel()) return;
    finish();

    parseGlobalAttributes( das, ddsVlist);
    if (cancelTask != null && cancelTask.isCancel()) return;

    // look for coordinate variables
    for (int i=0; i<variables.size(); i++) {
      Variable v = (Variable) variables.get(i);
      if (v instanceof DODSVariable)
        ((DODSVariable)v).calcIsCoordinateVariable();
    }

    // see if theres a CE: if so, we need to reset the dodsConnection without it,
    // since we are reusing dodsConnection; perhaps this is not needed?
    // may be true now that weve consolidated metadata reading
    // no comprende
    int pos;
    if (0 <= (pos = urlName.indexOf('?'))) {
      String datasetName = urlName.substring(0, pos);
      if (debugServerCall) System.out.println(" reconnect to = <"+datasetName+">");
      dodsConnection = new DConnect(datasetName, accept_deflate);

      // parse the CE for projections
      String CE = urlName.substring(pos+1);
      StringTokenizer stoke = new StringTokenizer( CE, " ,");
      while (stoke.hasMoreTokens()) {
        String proj = stoke.nextToken();
        int subsetPos = proj.indexOf('[');
        if (debugCE) System.out.println(" CE = "+proj+" "+subsetPos);
        if (subsetPos > 0) {
          String vname = proj.substring(0,subsetPos);
          String vCE = proj.substring(subsetPos);
          if (debugCE) System.out.println(" vCE = <"+vname+"><"+vCE+">");
          DODSVariable dodsVar = (DODSVariable) findVariable( vname); // LOOK ??
          dodsVar.setCE( vCE);
          dodsVar.setCaching( true);
        }
      }
    }

    // preload scalers, coordinate variables, strings, small arrays
    if (preload) {
      List preloadList = new ArrayList();
      for (int i=0; i <variables.size(); i++) {
        Variable dodsVar = (Variable) variables.get(i);
        if (dodsVar.getCoordinateDimension() != null || dodsVar.isCaching() || dodsVar.getDataType() == DataType.STRING) {
          dodsVar.setCaching( true);
          preloadList.add( dodsVar);
        }
      }
      if (cancelTask != null && cancelTask.isCancel()) return;
      readArrays( preloadList);
    }

    if (showNCfile) System.out.println("DODS nc file = "+this);
    ddsSave = dds;
    dasSave = das;
  }

  // parse the DDS, creating a tree of DodsV objects
  private ArrayList parseDDS(DDS dds) throws IOException {
    ArrayList dodsVlist  = new ArrayList();

    // recursively get the Variables from the DDS
    Enumeration variables = dds.getVariables();
    parseVariables( null, variables, dodsVlist);

    // assign depth first sequence number
    nextInSequence = 0;
    assignSequence( dodsVlist);
    return dodsVlist;
  }

  // DDS -> {BaseType} for arrays, BaseType = DArray -> {elemType}
  // here we 1) put all Variables into a DodsV, 2) unravel DConstructors (DSequence, DStructure, DGrid)
  // 3) for Darray, we put Variable = elemType, and store the darray seperately, not in the heirarchy.
  // so you need to get the parent from the dodsV.
  private void parseVariables( DodsV parent, Enumeration variables, ArrayList dodsVlist) {
    while (variables.hasMoreElements()) {
      dods.dap.BaseType bt = (dods.dap.BaseType) variables.nextElement();
      DodsV dodsV = new DodsV( parent, bt);
      dodsVlist.add(dodsV);
      if (bt instanceof DConstructor) {
        DConstructor dcon = (DConstructor) bt;
        java.util.Enumeration enumerate2 = dcon.getVariables();
        parseVariables( dodsV, enumerate2, dodsV.children);

      } else if (bt instanceof DArray) {
        DArray da = (DArray) bt;

        BaseType elemType = da.getPrimitiveVector().getTemplate();
        dodsV.bt = elemType;
        dodsV.darray = da;

        if (elemType instanceof DConstructor) { // note that for DataDDS, cant traverse this further to find the data.
          DConstructor dcon = (DConstructor) elemType;
          java.util.Enumeration nestedVariables = dcon.getVariables();
          parseVariables( dodsV, nestedVariables, dodsV.children);
        }
      }
    }
  }

  // assign depth first sequence number
  private int nextInSequence = 0;
  private void assignSequence( ArrayList dodsVlist) {
    for (int i = 0; i < dodsVlist.size(); i++) {
      DodsV dodsV =  (DodsV) dodsVlist.get(i);
      assignSequence( dodsV.children);
      dodsV.seq = nextInSequence;
      nextInSequence++;
    }
  }

  // parse the DAS, assign attrinute tables to the DodsV objects.
  // nestedd attribute tables acctuallly follow the tree we construct with dodsV, so its
  // easy to assign to correct dodsV.
  private void parseDAS(DAS das, ArrayList dodsVlist) throws IOException {
    // recursively find the Attributes from the DAS
    Enumeration tableNames = das.getNames();
    parseAttributes( das, tableNames, dodsVlist);
  }

  private void parseAttributes( DAS das, Enumeration tableNames, ArrayList dodsVlist) {
    while (tableNames.hasMoreElements()) {
      String tableName = (String) tableNames.nextElement();
      AttributeTable attTable = das.getAttributeTable( tableName);

      // see if there's a matching variable
      String name = attTable.getName();
      DodsV dodsV = findDodsV( name, dodsVlist, false); // short name matches the table name
      if (dodsV != null) {
        dodsV.attTable = attTable;
        if (debugAttributes) System.out.println("DODSNetcdf getAttributes found <"+name+"> :");
        continue;
      }
      if (debugAttributes) System.out.println("DODSNetcdf getAttributes CANT find <"+name+"> :");
    }
  }

  // search the list for a BaseType with given name
  private DodsV findDodsV( String name, ArrayList dodsVlist, boolean useDone ) {
    for (int i = 0; i < dodsVlist.size(); i++) {
      DodsV dodsV =  (DodsV) dodsVlist.get(i);
      if (useDone && dodsV.isDone) continue;
      if (name.equals( dodsV.bt.getName()))
        return dodsV;
    }
    return null;
  }

  // find the DodsV object in the dataVlist corresponding to the ddsV
  private DodsV findDataV( DodsV ddsV, ArrayList dataVlist ) {
    if (ddsV.parent != null) {
      DodsV parentV = findDataV( ddsV.parent, dataVlist);
      if (parentV == null) // dataDDS may not have the structure wrapper
        return findDodsV( ddsV.bt.getName(), dataVlist, true);
      return findDodsV( ddsV.bt.getName(), parentV.children, true);
    }

    DodsV dataV =  findDodsV( ddsV.bt.getName(), dataVlist, true);
    /* if ((dataV == null) && (ddsV.bt instanceof DGrid)) { // when asking for the Grid array
      DodsV gridArray = (DodsV) ddsV.children.get(0);
      return findDodsV( gridArray.bt.getName(), dataVlist, true);
    } */
    return dataV;
  }

  // a container for dods basetypes, so we can add some stuff as we process it
  class DodsV implements Comparable {
    //String name; // full name
    BaseType bt;
    DodsV parent;
    ArrayList children = new ArrayList();
    DArray darray; // if its an array
    AttributeTable attTable;
    Array data; // preload
    boolean isDone; // nc var has been made
    int seq; // "depth first" order

    DodsV( DodsV parent, BaseType bt) {
      this.parent = parent;
      this.bt = bt;
    }

    public int compareTo(Object o) {
       return seq - ((DodsV)o).seq;
    }
  }

  private void parseGlobalAttributes(DAS das, ArrayList dodsVlist) {
    // loop over attribute tables, collect global attributes
    java.util.Enumeration tableNames = das.getNames();
    while (tableNames.hasMoreElements()) {
      String tableName = (String) tableNames.nextElement();
      AttributeTable attTable = das.getAttributeTable( tableName);

      if (tableName.equals("NC_GLOBAL") || tableName.equals("HDF_GLOBAL")) {
        java.util.Enumeration attNames = attTable.getNames();
        while (attNames.hasMoreElements()) {
          String attName = (String) attNames.nextElement();
          dods.dap.Attribute att = attTable.getAttribute(attName);

          DODSAttribute ncatt = new DODSAttribute( attName, att);
          addAttribute( null, ncatt);
        }

      } else if (tableName.equals("DODS_EXTRA")) {
        java.util.Enumeration attNames = attTable.getNames();
        while (attNames.hasMoreElements()) {
          String attName = (String) attNames.nextElement();
          if (attName.equals("Unlimited_Dimension")) {
            dods.dap.Attribute att = null;
            //try {
              att = attTable.getAttribute(attName);
            //} catch (dods.dap.NoSuchAttributeException e) {
            //  continue;
            //}

            DODSAttribute ncatt = new DODSAttribute( attName, att);
            setUnlimited( ncatt.getStringValue());
          } else
            System.out.println(" HEY! unknown DODS_EXTRA attribute = "+ attName);
        }

      } else if (null == findDodsV( tableName, dodsVlist, false)) {
        addAttributes(attTable.getName(), attTable);
      }
    }
  }

  ///////////////////////////////////////////////////////////////////

  private void preload(ArrayList dodsVlist, CancelTask cancelTask) throws IOException {
    //Build up the request
    StringBuffer requestString = new StringBuffer ();
    requestString.append ( "?");
    ArrayList wantList = new ArrayList();
    preloadFindMaps( dodsVlist, requestString, wantList);

    if (wantList.size() == 0) return;

    try {
      ArrayList flatDataList = parseDDS( readDataDDSfromServer (requestString.toString()));
      if (cancelTask != null && cancelTask.isCancel()) return;

      for (int i = 0; i < wantList.size(); i++) {
        DodsV v = (DodsV) wantList.get(i);
        DodsV dataV = (DodsV) flatDataList.get(i);
        v.data = convertMapData( dataV.darray);
        if (cancelTask != null && cancelTask.isCancel()) return;
      }

     } catch (Exception exc) {
      System.err.println ("Error:" + exc);
      exc.printStackTrace ();
      throw new IOException( exc.getMessage());
    }
  }

  private void preloadFindMaps( ArrayList dodsVlist, StringBuffer result, ArrayList want) {
    for (int i = 0; i < dodsVlist.size(); i++) {
      DodsV dodsV =  (DodsV) dodsVlist.get(i);
      if (dodsV.bt instanceof DGrid) {
        List maps = dodsV.children;
        for (int j = 1; j < maps.size(); j++) {
          DodsV map = (DodsV) maps.get(j);
          if (want.size() > 0) result.append( ",");
          want.add( map);
          result.append( makeDODSname( map));
        }
      }
      // recurse
      if (dodsV.bt instanceof DStructure) {
        preloadFindMaps( dodsV.children, result, want);
      }
    }
  }

  private Array convertMapData( DArray da) {
      // gotta be a DVector with primitive type
    dods.dap.PrimitiveVector pv = da.getPrimitiveVector();
    BaseType elemType = pv.getTemplate();

    Object storage = pv.getInternalStorage();
    storage = widenArray( pv, storage); // data conversion if needed

     // create the array, using  DODS internal array so there's no copying
    return Array.factory( convertToNCType( elemType).getPrimitiveClassType(), makeShape( da), storage);
  }

  //////////////////////////////////////////////////////////////////////////////////////////////

  private void constructTopVariables(ArrayList dodsVlist, CancelTask cancelTask) throws IOException {
    for (int i = 0; i < dodsVlist.size(); i++) {
      DodsV dodsV =  (DodsV) dodsVlist.get(i);
      if (dodsV.bt instanceof DConstructor) continue;

      addVariable( rootGroup, null, dodsV);
      if (cancelTask != null && cancelTask.isCancel()) return;
    }
  }

  private void constructConstructors(ArrayList dodsVlist, CancelTask cancelTask) throws IOException {
    for (int i = 0; i < dodsVlist.size(); i++) {
      DodsV dodsV =  (DodsV) dodsVlist.get(i);
      if (dodsV.isDone) continue;

      addVariable( rootGroup, null, dodsV);
      if (cancelTask != null && cancelTask.isCancel()) return;
    }
  }

    // recursively make new variables: all new variables come through here
  Variable addVariable(Group parentGroup, Structure parentStructure, DodsV dodsV) throws IOException {
    Variable v = makeVariable( parentGroup, parentStructure, dodsV);
    if (v != null) {
      addAttributes( v, dodsV.attTable);
      if (parentStructure == null)
        parentGroup.addVariable( v);
      else
        parentStructure.addMemberVariable(v);

      v.setSPobject( dodsV);
      dodsV.isDone = true;
    }
    return v;
  }

  private Variable makeVariable(Group parentGroup, Structure parentStructure, DodsV dodsV) throws IOException {

    dods.dap.BaseType dodsBT = dodsV.bt;
    String name = dodsBT.getName();
    if (debugConstruct) System.out.print("DODSNetcdf makeVariable try to init <"+name+"> :");

    // arrays of Strings
    if (dodsBT instanceof DString) {
      if (dodsV.darray == null) {
        if (debugConstruct) System.out.println("  assigned to DString: name = "+name);
        return new DODSVariable( this, parentGroup, parentStructure, name, (DString) dodsBT);
      } else {
        if (debugConstruct) System.out.println("  assigned to Array of Strings: name = "+name);
        return new DODSVariable( this, parentGroup, parentStructure, name, dodsV.darray, (DString) dodsBT);
      }

    // arrays of primitives
    } else if ((dodsBT instanceof DBoolean) || (dodsBT instanceof DByte) ||
               (dodsBT instanceof DFloat32) || (dodsBT instanceof DFloat64) ||
               (dodsBT instanceof DInt16) || (dodsBT instanceof DInt32) ||
               (dodsBT instanceof DUInt16) || (dodsBT instanceof DUInt32)) {
      if (dodsV.darray == null) {
        if (debugConstruct) System.out.println("  assigned to scalar "+dodsBT.getTypeName()+": name = "+name);
        return new DODSVariable( this, parentGroup, parentStructure, name, dodsBT);
      } else {
        if (debugConstruct) System.out.println("  assigned to array of type "+dodsBT.getClass().getName()+": name = "+name);
        return new DODSVariable( this, parentGroup, parentStructure, name, dodsV.darray, dodsBT);
      }
    }

    if (dodsBT instanceof DGrid) {
      DGrid grid = (DGrid) dodsBT;
      if (dodsV.darray == null) {
        if (debugConstruct) System.out.println(" assigned to DGrid <"+name+">");

        //  common case is that the map vectors already exist as top level variables
        // this is how the netccdf servers do it
        for (int i = 1; i < dodsV.children.size(); i++) {
          DODSNetcdfFile.DodsV map = (DODSNetcdfFile.DodsV) dodsV.children.get(i);
          String shortName = map.bt.getName();
          Variable mapV = parentGroup.findVariable( shortName);
          if (mapV == null) {        // if not, add it LOOK need to compare values
            mapV = addVariable(parentGroup, parentStructure, map);
            makeCoordinateVariable( parentGroup, mapV, map.data);
          }
        }

        return new DODSGrid( this, parentGroup, parentStructure, name, dodsV);

      } else {
        if (debugConstruct) System.out.println(" ERROR! array of DGrid <"+name+">");
        return null;
      }

    } else if (dodsBT instanceof DSequence) {
      DSequence dseq = (DSequence) dodsBT;
      if (dodsV.darray == null) {
        if (debugConstruct) System.out.println(" assigned to DSequence <"+name+">");
        return new DODSStructure( this, parentGroup, parentStructure, name, dodsV);
      } else {
        if (debugConstruct) System.out.println(" ERROR! array of DSequence <"+name+">");
        return null;
      }

    } else if (dodsBT instanceof DStructure) {
      DStructure dstruct = (DStructure) dodsBT;
      if (dodsV.darray == null) {
        if (useGroups && (parentStructure == null) && isGroup(dstruct)) { // turn into a group
          if (debugConstruct) System.out.println(" assigned to Group <"+name+">");
          Group g = new Group(this, parentGroup, name);
          addAttributes( g, dodsV.attTable);
          parentGroup.addGroup( g);

          for (int i = 0; i < dodsV.children.size(); i++) {
            DodsV nested = (DodsV) dodsV.children.get(i);
            addVariable( g, null, nested);
          }
          return null;
        } else {
          if (debugConstruct) System.out.println(" assigned to DStructure <"+name+">");
          return new DODSStructure( this, parentGroup, parentStructure, name, dodsV);
        }
      } else {
        if (debugConstruct) System.out.println(" assigned to Array of DStructure <"+name+"> ");
        return new DODSStructure( this, parentGroup, parentStructure, name, dodsV.darray, dodsV);
      }

    } else {
      System.out.println("DODSNetcdf didnt process basetype <"+dodsBT.getTypeName()+"> name = "+name);
      return null;
    }

  }

  private void makeCoordinateVariable ( Group parentGroup, Variable v, Array data) {
    String name = v.getShortName();

    // replace in Variable
    Dimension oldDimension = v.getDimension(0);
    Dimension newDimension = new Dimension( name, oldDimension.getLength(), true);
    // newDimension.setCoordinateAxis( v); calcCoordinateVaribale will do this
    v.setDimension( 0, newDimension);

    // replace old (if it exists) in Group with shared dimension
    Dimension old = parentGroup.findDimension( name);
    parentGroup.remove( old);
    parentGroup.addDimension( newDimension);

    // might as well cache the data
    if (data != null) {
      v.setCachedData(data, false);
      if (debugCached) System.out.println(" cache for <"+name+"> length ="+ data.getSize());
    }
  }

  // make a structure into a group if its scalar and all parents are groups
  private boolean isGroup( DStructure dstruct) {
    BaseType parent = dstruct.getParent();
    if (parent == null) return true;
    if (parent instanceof DStructure)
      return isGroup((DStructure) parent);
    return true;
  }

  private void addAttributes(String tableName, AttributeTable attTable) {

    java.util.Enumeration attNames = attTable.getNames();
    while (attNames.hasMoreElements()) {
      String attName = (String) attNames.nextElement();
      dods.dap.Attribute att = attTable.getAttribute(attName);
      if (!att.isContainer()) {
        DODSAttribute ncatt = new DODSAttribute( tableName +"." + attName, att);
        addAttribute( null, ncatt);
        if (debugAttributes) System.out.println(" addAttribute = "+tableName +"." + attName);
      }
    }

    try {
      // look for nested ones
      attNames = attTable.getNames();
      while (attNames.hasMoreElements()) {
        String attName = (String) attNames.nextElement();
        dods.dap.Attribute att = attTable.getAttribute(attName);
        if (att.isContainer()) {
          addAttributes(tableName +"."+attName, att.getContainer());
        }
      }
    } catch (Exception e) {} // WRONG
    // } catch (NoSuchAttributeException e) {}
  }

  // an attTable name matches a Variable
  private void addAttributes(Variable v, AttributeTable attTable) {
    if (attTable == null) return;
    if (debugAttributes) System.out.println(" addAttributes to "+v.getName());

    java.util.Enumeration attNames = attTable.getNames();
    while (attNames.hasMoreElements()) {
      String attName = (String) attNames.nextElement();
      dods.dap.Attribute att = attTable.getAttribute(attName);
      if (att == null) continue;
      if (!att.isContainer()) {
        DODSAttribute ncatt = new DODSAttribute( attName, att);
        v.addAttribute( ncatt);
      } /* else if (v instanceof Structure) {
        Structure s = (Structure) v;
        Variable member = s.findVariable( attTable.getName());
        if (member != null) {
          addAttributes(member, att.getContainer());
        } else {
          System.out.println("Cant find nested Variable "+ attTable.getName()+" in "+v.getName());
        }
      } else {
        System.out.println("Container attribute "+ attTable.getName()+" in non structure variables"+v.getName());
      } */
    }
  }

  // an attTable name matches a Group
  private void addAttributes(Group g, AttributeTable attTable) {
    if (attTable == null) return;
    if (debugAttributes) System.out.println(" addAttributes to Group "+g.getName());

    java.util.Enumeration attNames = attTable.getNames();
    while (attNames.hasMoreElements()) {
      String attName = (String) attNames.nextElement();
      dods.dap.Attribute att = attTable.getAttribute(attName);
      if (att == null) continue;
      if (!att.isContainer()) {
        DODSAttribute ncatt = new DODSAttribute( attName, att);
        g.addAttribute( ncatt);
      } /* else {
        Variable member = g.findVariable( attTable.getName());
        if (member != null) {
          addAttributes(member, att.getContainer());
        } else {
          Group nested = g.findGroup( attTable.getName());
          if (nested != null) {
            addAttributes(nested, att.getContainer());
          } else {
            System.out.println("Cant find nested place for nested attribute "+ attTable.getName()+" in group "+g.getName());
          }
        }
      } */
    }
  }

  // construct list of dimensions to use
  protected ArrayList constructDimensions( Group group, dods.dap.DArray dodsArray) {
    if (group == null) group = rootGroup;

    ArrayList dims = new ArrayList();
    Enumeration enumerate = dodsArray.getDimensions();
    while (enumerate.hasMoreElements()) {
      dods.dap.DArrayDimension dad = (dods.dap.DArrayDimension) enumerate.nextElement();
      String name = dad.getName();

      Dimension myd = null;

      if (name == null) { // if no name, make an anonymous dimension
        myd = new Dimension( "", dad.getSize(), false);

      } else { // see if shared
        myd = group.findDimension(name);
        if (myd == null) { // add as shared
          myd = new Dimension( dad.getName(), dad.getSize(), true);
          group.addDimension( myd);
        } else if (myd.getLength() != dad.getSize()) { // make a non-shared dimension
          myd = new Dimension( name, dad.getSize(), false);
        } // else use existing, shared dimension
      }
      dims.add( myd); // add it to the list
    }

    return dims;
  }

  private void setUnlimited( String dimName) {
    Dimension dim = (Dimension) rootGroup.findDimension( dimName);
    if (dim != null)
      dim.setUnlimited(true);
    else
      System.out.println(" HEY! DODS Unlimited_Dimension = "+ dimName+" not found");
  }

  protected int[] makeShape( dods.dap.DArray dodsArray) {
    int count = 0;
    Enumeration enumerate = dodsArray.getDimensions();
    while (enumerate.hasMoreElements()) {
      count++;
      enumerate.nextElement();
   }

    int[] shape = new int[count];
    enumerate = dodsArray.getDimensions();
    count = 0;
    while (enumerate.hasMoreElements()) {
      dods.dap.DArrayDimension dad = (dods.dap.DArrayDimension) enumerate.nextElement();
      shape[count++] = dad.getSize();
    }

    return shape;
  }

  // kludge for single inheritence
  private String getDODSshortName( Variable var) {
    if (var instanceof DODSVariable)
      return ((DODSVariable)var).getDODSshortName();
    else if (var instanceof DODSStructure)
      return ((DODSStructure)var).getDODSshortName();
    else
      return null;
  }

  // full name
  private String makeDODSname(Variable dodsV) {
    if (dodsV.isMemberOfStructure())
      return makeDODSname(dodsV.getParentStructure()) + "." + getDODSshortName(dodsV);

    Group parent = dodsV.getParentGroup();
    if (parent != rootGroup)
      return parent.getShortName() + "." + getDODSshortName(dodsV);
    else
      return getDODSshortName(dodsV);
  }

  // full name
  private String makeDODSname(DodsV dodsV) {
    DodsV parent = dodsV.parent;
    if (null != parent)
      return ( makeDODSname(parent) + "."+dodsV.bt.getName());
    return dodsV.bt.getName();
  }

  /** Get the DODS data class corresponding to the Netcdf data type.
   *  This is the inverse of getNetCDFType().
   *  @param dataType Netcdf data type.
   *  @return the corresponding DODS type enum, from dods.dap.Attribute.XXXX.
   */
  static public int convertToDODSType(DataType dataType) {

    if (dataType == DataType.STRING)
      return dods.dap.Attribute.STRING;
    if (dataType == DataType.BYTE)
      return dods.dap.Attribute.BYTE;
    if (dataType == DataType.FLOAT)
      return dods.dap.Attribute.FLOAT32;
    if (dataType == DataType.DOUBLE)
      return dods.dap.Attribute.FLOAT64;
    if (dataType == DataType.SHORT)
      return dods.dap.Attribute.INT16;
    if (dataType == DataType.INT)
      return dods.dap.Attribute.INT32;
    if (dataType == DataType.BOOLEAN)
      return dods.dap.Attribute.BYTE;
    if (dataType == DataType.LONG)
      return dods.dap.Attribute.UINT32;

    // shouldnt happen
    return dods.dap.Attribute.STRING;
  }

  /** Get the Netcdf data type corresponding to the DODS data type.
   *  This is the inverse of getDODSType().
   *  @param dodsDataType DODS type enum, from dods.dap.Attribute.XXXX.
   *  @return the corresponding netcdf data class type.
   */
  static public DataType convertToNCType(int dodsDataType) {
    switch ( dodsDataType) {
      case dods.dap.Attribute.BYTE: return DataType.SHORT; // widen BYTE to short because DODS BYTE is unsigned
      case dods.dap.Attribute.FLOAT32: return DataType.FLOAT;
      case dods.dap.Attribute.FLOAT64: return DataType.DOUBLE;
      case dods.dap.Attribute.INT16: return DataType.SHORT;
      case dods.dap.Attribute.UINT16: return DataType.INT;
      case dods.dap.Attribute.INT32: return DataType.INT;
      case dods.dap.Attribute.UINT32: return DataType.LONG;
      default: return DataType.STRING;
    }
  }

     /** Assign a Java data type. */
  static public DataType convertToNCType(dods.dap.BaseType dtype) {

    if (dtype instanceof DString)
      return DataType.STRING;
    else if (dtype instanceof DFloat32)
      return DataType.FLOAT;
     else if (dtype instanceof DFloat64)
      return DataType.DOUBLE;
    else if (dtype instanceof DUInt32)
      return DataType.LONG; // widen
    else if (dtype instanceof DUInt16)
      return DataType.INT; // widen
    else if (dtype instanceof DInt32)
      return DataType.INT;
    else if (dtype instanceof DInt16)
      return DataType.SHORT;
    else if (dtype instanceof DByte)
      return DataType.BYTE;
    else if (dtype instanceof DBoolean)
      return DataType.BOOLEAN;
    else
      throw new IllegalArgumentException("DODSVariable illegal type = "+dtype.getTypeName());
  }

  /////////////////////////////////////////////////////////////////////////////////////

  /** This does the actual connection to the dods server and reading of the data.
   * All data calls go through here so we can add debugging.
   * @param CE constraint expression; use empty string if none
   * @return DataDDS
   */
  DataDDS readDataDDSfromServer(String CE) throws IOException, dods.dap.parser.ParseException,
       dods.dap.DODSException  {
    if (debugServerCall) System.out.println("DODSNetcdfFile.readFromServer = <"+CE+">");

    long start = 0;
    if (debugTime) start = System.currentTimeMillis();

    if (!CE.startsWith("?"))
       CE = "?" + CE;
    DataDDS data =  dodsConnection.getData(CE, null);

    if (debugTime) System.out.println("DODSNetcdfFile.readFromServer took = "+(System.currentTimeMillis()-start)/1000.0);

    if (debugDataResult) {
      System.out.println(" dataDDS return:");
      data.print(System.out);
    }

    return data;
  }

  /** close the dataset. */
  public void close() throws java.io.IOException {
    // dodsConnection.close(); LOOK why not?
    isClosed = true;
  }

  ///////////////////////////////////////////////////////////////////
  // ALL the I/O goes through these  routines
  // called from ucar.nc2.Variable

  /**
   * Make a single call to the DODS Server to read all the named variable's data
   * in one client/server roundtrip.
   *
   * @param preloadVariables list of of variables to preload
   * @return list of type Array
   * @throws IOException
   */
  public List readArrays (List preloadVariables) throws IOException {
    //For performance tests:
    //if (true) return super.readArrays (variables);
    if (preloadVariables.size() == 0) return new ArrayList();

    // construct the list of variables, skipping ones with cached data
    ArrayList reqDodsVlist = new ArrayList();
    ArrayList dataVlist = null;
    for (int i=0; i<preloadVariables.size(); i++) {
      Variable var = (Variable) preloadVariables.get (i);
      if (var.hasCachedData()) continue;
      reqDodsVlist.add( var.getSPobject());
    }
    Collections.sort(reqDodsVlist); // "depth first" order

    // read the data
    DataDDS dataDDS = null;
    HashMap map = new HashMap( 2 * reqDodsVlist.size()+1);
    if (reqDodsVlist.size() > 0) {

      // Create the request
      StringBuffer requestString = new StringBuffer ();
      for (int i=0; i<reqDodsVlist.size(); i++) {
        DodsV dodsV = (DodsV) reqDodsVlist.get (i);
        requestString.append (i == 0 ? "?" : ",");
        requestString.append (makeDODSname(dodsV));
      }

      try {
        dataDDS = readDataDDSfromServer (requestString.toString());
        dataVlist = parseDDS( dataDDS);

      } catch (Exception exc) {
        System.err.println ("Error:" + exc);
        exc.printStackTrace ();
        throw new IOException( exc.getMessage());
      }

      // gotta find the corresponding data in "depth first" order
      for (int i=0;i<reqDodsVlist.size (); i++) {
        DodsV ddsV = (DodsV) reqDodsVlist.get(i);

        if (ddsV.bt instanceof DGrid) // really want its child array
          ddsV = (DodsV) ddsV.children.get(0);

        DodsV dataV = findDataV( ddsV, dataVlist);
        if (dataV != null) {
          if (debugConvertData) System.out.println("readArray found "+makeDODSname(ddsV));
          dataV.isDone = true;
          map.put( ddsV, dataV); // thread safe!
        }
      }
    }

    // For each variable either extract the data or use cached data.
    List result = new ArrayList();
    for (int i=0;i<preloadVariables.size (); i++) {
      Variable var = (Variable) preloadVariables.get(i);
      if (var.hasCachedData()) {
        result.add (var.read());

      } else {
        Array data = null;
        DodsV ddsV = (DodsV) var.getSPobject();
        DodsV dataV = (DodsV) map.get( ddsV);
        if (dataV == null) {
          System.out.println("ERROR (DODSNetcdfFile.readArrays) cant find "+makeDODSname(ddsV)+" in dataDDS = ");
          dataDDS.print( System.out);
        } else {
          if (debugConvertData) System.out.println("readArray found "+makeDODSname(ddsV));
          dataV.isDone = true;

          data = convertD2N( dataV, var, null, false);
          if (var.isCaching()) {
            var.setCachedData( data, false);
            if (debugCached) System.out.println(" cache for <"+var.getName()+"> length ="+ data.getSize());
          }
        }
        result.add (data);
      }
    }
    return result;
  }

  // this is for reading top variables
  public Array readData(ucar.nc2.Variable v, List section) throws IOException, InvalidRangeException  {

    // LOOK: what if theres already a CE !!!!
    // create the constraint expression
    StringBuffer buff = new StringBuffer(100);
    buff.setLength(0);
    buff.append( getDODSshortName(v));

    // add the selector if not a Sequence
    if (!v.isUnknownLength())
      makeSelector(buff, section);

    Array data = null;
    try {
      ArrayList dataList = parseDDS( readDataDDSfromServer(buff.toString()));
      data = convertD2N( (DodsV) dataList.get(0), v, section, false); // can only be one
    }
    catch (DODSException ex) {
      ex.printStackTrace();
      throw new IOException( ex.getMessage());
    }
    catch (ParseException ex) {
      ex.printStackTrace();
      throw new IOException( ex.getMessage());
    }

    return data;
  }

  // this is for reading variables that are members of structures
  public Array readMemberData(ucar.nc2.Variable v, List section, boolean flatten) throws IOException, InvalidRangeException  {
    StringBuffer buff = new StringBuffer(100);
    buff.setLength(0);

    // add the selector
    addParents( buff, v, section, 0);

    Array data = null;
    try {
      ArrayList dataList = parseDDS( readDataDDSfromServer(buff.toString()));
      data = convertD2N((DodsV) dataList.get(0), v, section, flatten); // can only be one
    }
    catch (DODSException ex) {
      ex.printStackTrace();
      throw new IOException( ex.getMessage());
    }
    catch (ParseException ex) {
       ex.printStackTrace();
     throw new IOException( ex.getMessage());
    }

    return data;
  }

  private int addParents(StringBuffer buff, Variable s, List section, int start) {
    Structure parent = s.getParentStructure();
    if (parent != null) {
      start = addParents(buff, parent, section, start);
      buff.append(".");
    }

    List subSection = section.subList(start, start+s.getRank());

    buff.append(getDODSshortName( s));

    if (!s.isUnknownLength()) // have to get the whole thing for a sequence !!
      makeSelector( buff, subSection);

    return start+s.getRank();
  }

  private void makeSelector( StringBuffer buff, List section) {
    for (int d=0; d<section.size(); d++) {
      Range r = (Range) section.get(d);
      buff.append("[");
      buff.append(r.first());
      buff.append(':');
      buff.append(r.stride());
      buff.append(':');
      buff.append(r.last());
      buff.append("]");
    }
  }


  ///////////////////////////////////////////////////////////////////////////////////////
  // convert DODS data structures to NetCDF data Structures

  /**
   * Covert DODS BaseType into a netCDF Array
   * @param dodsV data to convert is contained in this
   * @param ncVar this is the Variable we are reading
   * @param sectionAll the requested section, including parents, or null for all
   * @param flatten whether to unwrap parent structures, only used if its a member variable.
   * @return Array
   */
  private Array convertD2N( DodsV dodsV, Variable ncVar, List sectionAll, boolean flatten) {

    // start with the top variable
    Variable topVar = ncVar;
    while (topVar.isMemberOfStructure()) {
      topVar = topVar.getParentStructure();
    }

    // extract top dodsVar from dataDDS
    BaseType dodsVar = dodsV.darray == null ? dodsV.bt : dodsV.darray;
    /*try {
      dodsVar = dds.getVariable( getDODSshortName( topVar));
    } catch (NoSuchVariableException e) {
      // can happen when the f**ing thing is a Grid
      dodsVar = dds.getVariable( getDODSshortName( ncVar));
      topVar = ncVar;
    } */

    // recursively extract the data
    Array data = convertData( dodsVar, topVar);

    // flatten if needed
    if (flatten && ncVar.isMemberOfStructure()) {

      // deal with sequences
      Variable v = ncVar;
      boolean isSequence = v.isUnknownLength();
      while (v.isMemberOfStructure()) {
        v = v.getParentStructure();
        if (v.isUnknownLength()) isSequence = true;
      }

      int[] shape = Range.getShape( sectionAll);
      if (isSequence) {
        ArrayList shapeList = new ArrayList();
        addShapes( shapeList, data, ncVar.getName(), null);
        shape = Range.getShape(shapeList);
      }

      Array flatData = Array.factory( ncVar.getDataType().getPrimitiveClassType(), shape);
      IndexIterator flatIterator = flatData.getIndexIterator();

      flattenData( data, flatIterator, ncVar.getName());
      data = flatData;
    }

    return data;
  }

  // LOOK nested sequences cant be flattened, because they may have different lengths !!
  private void addShapes( ArrayList shapes, Array data, String varName, String sdName) {
    int[] shape = data.getShape();
    shapes.addAll( Range.factory(shape));

    if (varName.equals(sdName)) return; // done

    Class elemClass = data.getElementType();
    if (elemClass.equals( DataType.STRUCTURE.getPrimitiveClassType())) {
      StructureData ds = (StructureData) data.getObject( data.getIndex());
      StructureData.Member m = (StructureData.Member) ds.getMembers().get(0); // can only be one
      addShapes( shapes, m.getData(), varName, ds.getName());
    }
  }

  private void flattenData( Array data, IndexIterator flatIterator, String varName) {
    IndexIterator ii = data.getIndexIterator();
    Class elemClass = data.getElementType();

    if (elemClass.equals( DataType.STRUCTURE.getPrimitiveClassType())) {
      while (ii.hasNext()) {
        StructureData ds = (StructureData) ii.getObjectNext();
        String sdName = ds.getName();
        StructureData.Member m = (StructureData.Member) ds.getMembers().get(0); // can only be one
        if (sdName.equals( varName))
          flatIterator.setObjectNext( ds); // this is the target variable, dont recurse!
        else
          flattenData( m.getData(), flatIterator, varName); // recurse
      }
    }

    else if (elemClass.equals( DataType.STRING.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setObjectNext( ii.getObjectNext());
    }

    else if (elemClass.equals( DataType.DOUBLE.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setDoubleNext( ii.getDoubleNext());
    }

    else if (elemClass.equals( DataType.FLOAT.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setFloatNext( ii.getFloatNext());
    }

    else if (elemClass.equals( DataType.INT.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setIntNext( ii.getIntNext());
    }

    else if (elemClass.equals( DataType.SHORT.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setShortNext( ii.getShortNext());
    }

    else if (elemClass.equals( DataType.LONG.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setLongNext( ii.getLongNext());
    }

    else if (elemClass.equals( DataType.BYTE.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setByteNext( ii.getByteNext());
    }

    else if (elemClass.equals( DataType.BOOLEAN.getPrimitiveClassType())) {
      while (ii.hasNext())
        flatIterator.setBooleanNext( ii.getBooleanNext());
    }

    else throw new IllegalStateException( "DODSNetcdfFile flattenData "+elemClass.getName());

  }

  // convert dodsVar to Array corresponding to the ncVar
  private Array convertData(dods.dap.BaseType dodsVar, Variable ncVar) {
    if (dodsVar instanceof DSequence) {
      DSequence ds = (DSequence) dodsVar;
      int nrows = ds.getRowCount();

      // make the result array
      int[] shape = new int[1];
      shape[0] = nrows;
      Array array = Array.factory( DataType.STRUCTURE.getPrimitiveClassType(), shape);
      IndexIterator ii = array.getIndexIterator();

      // populate it
      for (int row=0; row<nrows; row++) {
        Vector v = ds.getRow(row);
        StructureData sd = convertStructureData(v.elements(), (Structure) ncVar);
        ii.setObjectNext(sd);
      }
      return array;
    }

    if (dodsVar instanceof DStructure) { // scalar structure
      Array array = Array.factory( DataType.STRUCTURE.getPrimitiveClassType(), new int[0]);
      IndexIterator ii = array.getIndexIterator();
      DStructure ds = (DStructure) dodsVar;
      StructureData sd = convertStructureData( ds.getVariables(), (Structure) ncVar);
      ii.setObjectNext(sd);
      return array;
    }

    if (dodsVar instanceof DGrid) { // scalar grid
      DGrid ds = (DGrid) dodsVar;
      try {
        DArray da = (DArray) ds.getVariable(getDODSshortName( ncVar));
        return convertArray(da, ncVar);
      } catch (NoSuchVariableException e) {
        e.printStackTrace();
        return null;
      }
    }

    if (dodsVar instanceof DArray) {
      DArray da = (DArray) dodsVar;
      return convertArray(da, ncVar);

    } else {

      return convertScalar(dodsVar, ncVar.getDataType());
    }

  }

  private Array convertArray(DArray da, Variable ncVar) {
    BaseType elemType = da.getPrimitiveVector().getTemplate();
    if (debugConvertData) System.out.println("  DArray type "+ elemType.getClass().getName());

    if (elemType instanceof DStructure) {  // array of structures LOOK no array of DGrid
      Array array = Array.factory( DataType.STRUCTURE.getPrimitiveClassType(), makeShape( da));
      IndexIterator ii = array.getIndexIterator();
      BaseTypePrimitiveVector pv = (BaseTypePrimitiveVector) da.getPrimitiveVector();
      for (int i=0; i<pv.getLength(); i++) {
        BaseType bt = pv.getValue(i);
        DStructure ds = (DStructure) bt;
        StructureData sd = convertStructureData( ds.getVariables(), (Structure) ncVar);
        ii.setObjectNext(sd);
      }
      return array;

    } else if (elemType instanceof DString) {
      return convertStringArray(da, ncVar);

    } else {

       // otherwise gotta be a DVector with primitive type
      dods.dap.PrimitiveVector pv = da.getPrimitiveVector();
      Object storage = pv.getInternalStorage();
      storage = widenArray( pv, storage); // data conversion if needed

       // create the array, using  DODS internal array so there's no copying
      return Array.factory( ncVar.getDataType().getPrimitiveClassType(), makeShape( da), storage);
    }
  }

    // convert a DODS scalar value to a netcdf Array
  private Array convertScalar (BaseType dodsScalar, DataType dataType) {;
    Array scalarData = Array.factory( dataType.getPrimitiveClassType(), new int[0]);
    Index scalarIndex = scalarData.getIndex();

    // set the data value, using scalarIndex from Variable
    if (dodsScalar instanceof DString) {
      String sval = ((DString)dodsScalar).getValue();
      scalarData.setObject( scalarIndex, sval);

    } else if (dodsScalar instanceof DUInt32) {
      int ival = ((DUInt32)dodsScalar).getValue();
      long lval = unsignedIntToLong( ival);        // LOOK unsigned
      scalarData.setLong( scalarIndex, lval);

    } else if (dodsScalar instanceof DUInt16) {
      short sval = ((DUInt16)dodsScalar).getValue();
      int ival = unsignedShortToInt(sval);
      scalarData.setInt( scalarIndex, ival);

    } else if (dataType == DataType.FLOAT)
      scalarData.setFloat( scalarIndex, ((DFloat32)dodsScalar).getValue());
    else if (dataType == DataType.DOUBLE)
      scalarData.setDouble( scalarIndex, ((DFloat64)dodsScalar).getValue());
    else if (dataType == DataType.INT)
      scalarData.setInt( scalarIndex, ((DInt32)dodsScalar).getValue());
    else if (dataType == DataType.SHORT)
      scalarData.setShort( scalarIndex, ((DInt16)dodsScalar).getValue());
    else if (dataType == DataType.BYTE)
      scalarData.setByte( scalarIndex, ((DByte)dodsScalar).getValue());
    else if (dataType == DataType.BOOLEAN)
      scalarData.setBoolean( scalarIndex, ((DBoolean)dodsScalar).getValue());
    else
      throw new IllegalArgumentException("DODSVariable extractScalar invalid type = "+dataType);

    return scalarData;
  }

  StructureData convertStructureData(Enumeration dodsVariables, Structure s) {
    StructureData structureData = new StructureData(s);

    while (dodsVariables.hasMoreElements()) {
      dods.dap.BaseType bt = (dods.dap.BaseType) dodsVariables.nextElement();
      if (debugConvertData) System.out.println("  convertStructureData member: " + bt.getName());
      Variable v = s.findVariable(bt.getName());
      if (v == null)
        throw new IllegalStateException("cant find member <"+bt.getName()+"> in structure = "+s.getName());
      Array data = convertData(bt, v);
      structureData.addMember(v, data);
    }
    return structureData;
  }

  private Array convertStringArray(DArray dv, Variable ncVar) {
    dods.dap.PrimitiveVector pv = dv.getPrimitiveVector();
    BaseTypePrimitiveVector btpv = (BaseTypePrimitiveVector) pv;
    int nStrings = btpv.getLength();
    String[] storage = new String[nStrings];
    int max_len = 0;
    for (int i=0; i<nStrings; i++) {
      BaseType bb = btpv.getValue(i);
      storage[i] = ((DString)bb).getValue();
      max_len = Math.max( max_len, storage[i].length());
    }

    // LOOK deal with length=1 barfalloney
    if (max_len == 1) {
      int[] shape = ncVar.getShape();
      int rank = shape.length;
      int extraDimSize = shape[rank-1];
      int newSize = (int)ncVar.getSize()/extraDimSize;
      String[] newStorage = new String[newSize];

      // merge last dimension
      StringBuffer sbuff = new StringBuffer();
      int newCount = 0;
      while (newCount < newSize) {
        int mergeCount = 0;
        sbuff.setLength(0);
        while (mergeCount < extraDimSize) {
          String s = storage[extraDimSize * newCount + mergeCount];
          if (s.length() == 0) break;
          sbuff.append( s);
          mergeCount++;
        }
        newStorage[ newCount++] = sbuff.toString();
      }

      // adjust the dimensions
      List dims = ncVar.getDimensions();
      ncVar.setDimensions( dims.subList(0, rank-1));
      int[] newShape = ncVar.getShape();
      return Array.factory( DataType.STRING.getPrimitiveClassType(), newShape, newStorage);
    }

    else
      return Array.factory( DataType.STRING.getPrimitiveClassType(), makeShape( dv), storage);
  }

  protected Object widenArray( PrimitiveVector pv, Object storage) {
    if (pv instanceof UInt32PrimitiveVector) {
      UInt32PrimitiveVector org = (UInt32PrimitiveVector) pv;
      int len = pv.getLength();
      long [] lpv = new long[len];
      for (int i=0; i<len; i++)
        lpv[i] = unsignedIntToLong( org.getValue(i));
      storage = lpv;

    } else if (pv instanceof UInt16PrimitiveVector) {
      UInt16PrimitiveVector org = (UInt16PrimitiveVector) pv;
      int len = pv.getLength();
      int [] ipv = new int[len];
      for (int i=0; i<len; i++)
        ipv[i] = unsignedShortToInt( org.getValue(i));
      storage = ipv;
    }

    return storage;
  }

  /** widen an unsigned int to a long */
  static long unsignedIntToLong(int i) {
    return (i < 0) ? (long) i + 4294967296L : (long) i;
  }

  /** widen an unsigned short to an int */
   static int unsignedShortToInt(short s) {
     return (s < 0) ? (int) s + 65536 : (int) s;
   }

  /** widen an unsigned byte to a short */
   static short unsignedByteToShort(byte b) {
     return (short) ((b < 0) ? (short) b + (short) 256 : (short) b);
   }

    ////////////////////////////////////////////////////////////////////////////////
  // debugging

  /**
   *
   * @return the underlying dods info for debugging
   */
  public String toStringDebug() {
    StringBuffer buff = new StringBuffer(2000);
    buff.setLength(0);
    buff.append("=======================================\nNETCDF DODS file "+getLocation()+"\n");

    buff.append("DDS = \n");
    ByteArrayOutputStream buffOS = new ByteArrayOutputStream(8000);
    ddsSave.print( buffOS);
    buff.append(buffOS.toString());

    buff.append("===================\nDAS = \n");
    buffOS = new ByteArrayOutputStream(8000);
    dasSave.print( buffOS);
    buff.append(buffOS.toString());
    buff.append("===================\n");

    return buff.toString();
  }

  public static void main( String arg[]){
    //String url = "http://eosdata.gsfc.nasa.gov/daac-bin/nph-hdf/DODS/catalog/health/modis/L3ocean/hdf/MO1DMWD2.sst4.ADD2000297.002.2000366024147.hdf";
    // String url = (arg.length > 1) ? arg[0] : "http://motherlode.ucar.edu/cgi-bin/dods/DODS-3.2.1/nph-dods/dods/model/2003020200_sst-t.nc";
    //String url = "http://motherlode.ucar.edu/cgi-bin/dods/DODS-3.2.1/nph-dods/dods/model/example.nc";
    //String url = "http://dods.coas.oregonstate.edu:8080/dods/dts/test.21";
    //String url = "http://dods.coas.oregonstate.edu:8080/dods/dts/test.06a";
    String url = "dods://dods.ndbc.noaa.gov/cgi-bin/nph-nc/dods/burl1_2002_10_stdmet.nc";

    // "http://ingrid.ldeo.columbia.edu/expert/SOURCES/.LEVITUS94/dods";
    try {
      DODSNetcdfFile df = new DODSNetcdfFile(url, null);
      System.out.println("dods file = "+url+"\n"+df);
    } catch (Exception ioe) {
      System.out.println("error = "+url);
      ioe.printStackTrace();
    }
  }

}

/* Change History:
   $Log: DODSNetcdfFile.java,v $
   Revision 1.25  2004/12/10 17:04:17  caron
   *** empty log message ***

   Revision 1.24  2004/12/09 00:17:32  caron
   *** empty log message ***

   Revision 1.23  2004/12/08 18:08:31  caron
   implement _CoordinateAliasForDimension

   Revision 1.22  2004/12/07 02:43:22  caron
   *** empty log message ***

   Revision 1.21  2004/12/07 01:29:30  caron
   redo convention parsing, use _Coordinate encoding.

   Revision 1.20  2004/12/01 05:53:41  caron
   ncml pass 2, new convention parsing

   Revision 1.19  2004/11/18 23:24:58  caron
   add sequence iterator to DODSequence

   Revision 1.18  2004/11/10 17:01:35  caron
   redo Grids

   Revision 1.17  2004/11/07 03:00:50  caron
   *** empty log message ***

   Revision 1.16  2004/11/07 02:55:11  caron
   no message

   Revision 1.15  2004/11/04 00:38:18  caron
   no message

   Revision 1.14  2004/10/29 00:14:10  caron
   no message

   Revision 1.13  2004/10/19 19:46:17  caron
   change if dimensions are shared

   Revision 1.12  2004/10/06 19:03:42  caron
   clean up javadoc
   change useV3 -> useRecordsAsStructure
   remove id, title, from NetcdfFile constructors
   add "in memory" NetcdfFile

   Revision 1.11  2004/09/30 20:48:24  caron
   GDS grids

   Revision 1.10  2004/09/28 21:26:34  caron
   len=1 String, global attributes

   Revision 1.9  2004/09/22 13:46:37  caron
   *** empty log message ***

   Revision 1.8  2004/08/26 17:55:08  caron
   no message

   Revision 1.7  2004/08/19 21:38:12  caron
   no message

   Revision 1.6  2004/08/18 19:56:43  caron
   2.2 alpha (2)

   Revision 1.5  2004/08/17 19:20:06  caron
   2.2 alpha (2)

   Revision 1.4  2004/08/16 20:53:51  caron
   2.2 alpha (2)

   Revision 1.3  2004/07/12 23:40:18  caron
   2.2 alpha 1.0 checkin

   Revision 1.2  2004/07/06 19:28:12  caron
   pre-alpha checkin

   Revision 1.1.1.1  2003/12/04 21:05:28  caron
   checkin 2.2
 */