Changeset 3804

Timestamp:

Oct 4, 2007, 2:25:41 PM (16 years ago)

Author:

Nicklas Nordborg

Message:

References #721: Most things for attaching files to array designs are now in place.

• Added update script for updating existing installations
• Re-coded parts of the code that depended on depracated methods
• Changed depracated methods to be backwards compatible
• Plugins that import to or use array designs can now use the attached files

Location:

branches/filedb

Files:

• src/core/common-queries.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/Affymetrix.java (modified) (5 diffs)
• src/core/net/sf/basedb/core/ArrayDesign.java (modified) (10 diffs)
• src/core/net/sf/basedb/core/FeatureBatcher.java (modified) (3 diffs)
• src/core/net/sf/basedb/core/FileSet.java (modified) (3 diffs)
• src/core/net/sf/basedb/core/FileSetMember.java (modified) (1 diff)
• src/core/net/sf/basedb/core/FileStoreUtil.java (added)
• src/core/net/sf/basedb/core/Install.java (modified) (1 diff)
• src/core/net/sf/basedb/core/RawBioAssay.java (modified) (1 diff)
• src/core/net/sf/basedb/core/Update.java (modified) (9 diffs)
• src/plugins/core/net/sf/basedb/plugins/CdfFileReporterImporter.java (modified) (10 diffs)
• src/plugins/core/net/sf/basedb/plugins/PrintMapFlatFileImporter.java (modified) (8 diffs)
• src/plugins/core/net/sf/basedb/plugins/ReporterMapFlatFileImporter.java (modified) (8 diffs)
• src/test/TestArrayDesign.java (modified) (4 diffs)
• www/common/datafiles/list_files.jsp (modified) (2 diffs)

branches/filedb/src/core/common-queries.xml

@@ -2857 +2857 @@
     </description>
   </query>
+  
+  <query id="GET_FILESETMEMBER_FOR_DATAFILETYPE" type="HQL">
+    <sql>
+      SELECT mbr
+      FROM FileSetMemberData mbr
+      WHERE mbr.dataFileType.externalId = :externalId
+      AND mbr.fileSet = :fileSet
+    </sql>
+    <description>
+      A Hibernate query that loads a member in a given file set
+      if we know the external ID of the data file type.
+    </description>
+  </query>
+
+  
+  <query id="DELETE_PROPERTY_FILTER" type="SQL">
+    <sql>
+      DELETE FROM [PropertyFilters]
+      WHERE [property] = :property
+    </sql>
+    <description>
+      An SQL query that delete all PropertyFilters
+      which filter on a given property.
+    </description>
+  </query>
+  
+  <query id="GET_PLUGINS_WITH_KEYS" type="HQL">
+    <sql>
+      SELECT DISTINCT pk.pluginDefinitionId
+      FROM PluginKeys pk WHERE pk.keyId IN (:keys)
+    </sql>
+    <description>
+      A HQL query that has been granted a specific permission. Used by 
+      update scripts when new item types are added to make sure existing 
+      plug-ins can continue to work.
+    </description>
+  </query>
 
 

branches/filedb/src/core/net/sf/basedb/core/Affymetrix.java

@@ -79 +79 @@
       Affymetrix chip or if the file is not a CDF file
     @throws BaseException If there is another error
-    @deprecated
+    @deprecated Use {@link FileStoreUtil#setDataFile(DbControl, FileStoreEnabled, String, File)}
+      instead with {@link DataFileType#AFFYMETRIX_CDF} as the file
+      type
   */
   public static void setCdfFile(ArrayDesign design, File file)
@@ -85 +87 @@
   {
     if (design == null) throw new InvalidUseOfNullException("design");
-    if (!design.isAffyChip()) 
+    if (!design.isPlatform(Platform.AFFYMETRIX)) 
     {
       throw new InvalidDataException("Array design isn't an Affymetrix chip: " + design);
     }
-    design.checkPermission(Permission.WRITE);
     DbControl dc = design.getDbControl();
-    if (file != null) 
-    {
-      file.checkPermission(Permission.USE);
-      
-      // Verify that the file is a CDF file
-      FusionCDFData cdf = loadCdfFile(file);
-    
-      AnyToAny ata = AnyToAny.getNewOrExisting(dc, design, CDF_LINK_NAME, file, true);
-      if (!ata.isInDatabase()) dc.saveItem(ata);
-      design.getData().setHasFeatures(true);
-      design.getData().setNumFileFeatures(cdf.getHeader().getNumProbeSets());
-    }
-    else
-    {
-      AnyToAny.unlinkFrom(dc, design, CDF_LINK_NAME);
-      design.getData().setHasFeatures(false);
-    }
+    FileStoreUtil.setDataFile(dc, design, DataFileType.AFFYMETRIX_CDF, file);
+    if (file != null) design.getFileSet().validate(dc, true);
   }
   
@@ -131 +117 @@
       Affymetrix chip
     @throws BaseException If there is another error
-    @deprecated Use {@link FileSet#getMember(DataFileType)}
+    @deprecated Use {@link FileStoreUtil#getDataFile(DbControl, FileStoreEnabled, String)}
       with a data file type of {@link DataFileType#AFFYMETRIX_CDF}
       instead
@@ -139 +125 @@
   {
     if (design == null) throw new InvalidUseOfNullException("design");
-    if (!design.isAffyChip()) 
-    {
-      throw new InvalidDataException("Array design isn't an Affymetrix chip: " + design);
+    if (!design.isPlatform(Platform.AFFYMETRIX)) 
+    {
+      throw new InvalidDataException("Array design isn't an Affymetrix chip: " + design.getName());
     }
     DbControl dc = design.getDbControl();
-    File cdfFile = null;
-    if (design.hasFileSet())
-    {
-      FileSet fileSet = design.getFileSet();
-      DataFileType cdfType = DataFileType.getByExternalId(dc, DataFileType.AFFYMETRIX_CDF);
-      if (fileSet.hasMember(cdfType))
-      {
-        FileSetMember member = fileSet.getMember(cdfType);
-        cdfFile = member.getFile();
-      }
-    }
-    return cdfFile;
+    return FileStoreUtil.getDataFile(dc, design, DataFileType.AFFYMETRIX_CDF);
   }
 
@@ -211 +186 @@
           " must have an array design to set CEL file");
       }
-      if (!design.isAffyChip())
+      if (!design.isPlatform(Platform.AFFYMETRIX))
       {
         throw new InvalidDataException("The array design " + design.getName() + 

branches/filedb/src/core/net/sf/basedb/core/ArrayDesign.java

@@ -89 +89 @@
   }
 
+  /**
+    Create a new <code>ArrayDesign</code> item for a given platform.
+  
+    @param dc The <code>DbControl</code> which will be used for
+      permission checking and database access
+    @param platform The platform of the array design
+    @return The new <code>ArrayDesign</code> item
+    @throws BaseException If there is an error
+  */
   public static ArrayDesign getNew(DbControl dc, Platform platform)
   {
@@ -97 +106 @@
   }
 
+  /**
+    Create a new <code>ArrayDesign</code> item for a given platform
+    variant.
+  
+    @param dc The <code>DbControl</code> which will be used for
+      permission checking and database access
+    @param variant The platform variant of the array design
+    @return The new <code>ArrayDesign</code> item
+    @throws BaseException If there is an error
+  */
   public static ArrayDesign getNew(DbControl dc, PlatformVariant variant)
   {
@@ -104 +123 @@
     return ad;
   }
-  
   
   /**
@@ -273 +291 @@
   // -------------------------------------------
   
+  /**
+    Set the platform of the array design. This method will set the variant
+    to null. Use {@link #setVariant(PlatformVariant)} if you want to
+    set a specific variant.
+    @param platform The new platform
+    @throws PermissionDeniedException If the logged in user doesn't have
+      write permission
+    @throws InvalidDataException If platform is null
+  */
   public void setPlatform(Platform platform)
   {
@@ -281 +308 @@
   }
   
+  /**
+    Set the platform and variant of the array design. This method will 
+    automatically set the platform to {@link PlatformVariant#getPlatform()}.
+
+    @param variant The new platform variant, or null to remove
+      the variant and keep the platform as it is
+    @throws PermissionDeniedException If the logged in user doesn't have
+      write permission
+  */
   public void setVariant(PlatformVariant variant)
   {
@@ -296 +332 @@
   
   /**
-      @return TRUE if this is an AffyChip, FALSE otherwise.
-      @deprecated Use {@link #getPlatform()} instead and compare the
-        external ID with {@link Platform#AFFYMETRIX}
+    Check if the platform/variant of this array design has the
+    given external ID. 
+    @param externalId The external ID to match
+    @return TRUE if either the variant of platform matches the
+      external ID
+  */
+  public boolean isPlatform(String externalId)
+  {
+    if (externalId == null) return false;
+    if (externalId.equals(getData().getPlatform().getExternalId())) return true;
+    if (getData().getVariant() != null)
+    {
+      if (externalId.equals(getData().getVariant().getExternalId())) return true;
+    }
+    return false;
+  }
+  
+  /**
+    Check if this array design uses a file-only platform or variant.
+    If the array design has a platform variant, the setting from the 
+    variant is returned, otherwise the setting from the platform.
+    @return TRUE if this array design use a file-only platform
+  */
+  public boolean isFileOnlyPlatform()
+  {
+    if (getData().getVariant() != null)
+    {
+      return getData().getVariant().isFileOnly();
+    }
+    else
+    {
+      return getData().getPlatform().isFileOnly();
+    }
+  }
+  
+  /**
+      @return TRUE if this is an Affymetrix chip, FALSE otherwise.
+      @deprecated Use {@link #isPlatform(String)} instead with
+        {@link Platform#AFFYMETRIX} as the parameter
   */
   public boolean isAffyChip()
   {
-    return Platform.AFFYMETRIX.equals(getData().getPlatform().getExternalId());
-  }
-
-  /**
+    return isPlatform(Platform.AFFYMETRIX);
+  }
+
+  /**
+    Check if this array design has information about features, either
+    in the database or in files.
       @return TRUE if this object has features, FALSE otherwise
   */
@@ -491 +565 @@
     that can add features to the new block.
     <p>
-    It is not possible to add blocks to an Affymetrix chip. It stores
-    it's feature in a CDF file instead. See {@link Affymetrix#setCdfFile(ArrayDesign, File)}.
+    It is not possible to add blocks to an array design which uses a
+    file-only platform. The must store feature information in files 
+    instead. See {@link Platform} and {@link DataFileType}.
 
     @param bi The information for the new block
@@ -503 +578 @@
     throws PermissionDeniedException, BaseException
   {
+    if (getData().getPlatform().isFileOnly()) 
+    {
+      throw new PermissionDeniedException("Can't add blocks to a file-only array design: " + this.getName());
+    }
     if (hasFeatures()) 
     {
-      throw new PermissionDeniedException("Can't add blocks to a design that already has features: " + this);
-    }
-    if (isAffyChip()) 
-    {
-      throw new PermissionDeniedException("Can't add blocks to an Affymetrix chip: " + this);
+      throw new PermissionDeniedException("Can't add blocks to a design that already has features: " + this.getName());
     }
     checkPermission(Permission.WRITE);
@@ -531 +606 @@
     array design.   
     <p>
-    It is not possible to add features to an Affymetrix chip. It stores
-    it's feature in a CDF file instead. See {@link Affymetrix#setCdfFile(ArrayDesign, File)}.
+    It is not possible to add blocks to an array design which uses a
+    file-only platform. The must store feature information in files 
+    instead. See {@link Platform} and {@link DataFileType}.
 
     @throws PermissionDeniedException If raw data has already been added
@@ -542 +618 @@
   {
     checkPermission(Permission.WRITE);
+    if (isFileOnlyPlatform()) 
+    {
+      throw new PermissionDeniedException("Can't add features to a file-only array design: " + this.getName());
+    }
     if (hasFeatures()) 
     {
-      throw new PermissionDeniedException("Features has already been added to "+this);
-    }
-    if (isAffyChip()) 
-    {
-      throw new PermissionDeniedException("Can't add features to an Affymetrix chip: " + this);
+      throw new PermissionDeniedException("Features has already been added to "+this.getName());
     }
     if (featureBatcher == null)

branches/filedb/src/core/net/sf/basedb/core/FeatureBatcher.java

@@ -77 +77 @@
   
   /**
-    If the array design is an affy chip which needs AffyFeatureData
-    objects instead of FeatureData.
-  */
-  private final boolean isAffy;
-
-  
-  /**
     Keeps track of the position number.
   */
@@ -94 +87 @@
     this.arrayDesign = arrayDesign;
     this.arrayDesignData = arrayDesign.getData();
-    this.isAffy = arrayDesign.isAffyChip();
     setDbControl(dc);
   }
@@ -209 +201 @@
     throws BaseException
   {
-    if (getTotalInsertCount() > 0)
+    int insertCount = getTotalInsertCount();
+    if (insertCount > 0)
     {
       arrayDesignData.setHasFeatures(true);
+      arrayDesignData.setNumDbFeatures(insertCount);
     }
   }

branches/filedb/src/core/net/sf/basedb/core/FileSet.java

@@ -255 +255 @@
   }
   
+  public ItemQuery<FileSetMember> getMembers(String genericType)
+  {
+    ItemQuery<FileSetMember> query = getMembers();
+    query.restrictPermanent(
+      Restrictions.eq(
+        Hql.property("dataFileType.genericType.systemId"), 
+        Expressions.string(genericType)
+      )
+    );
+    return query;
+  }
+  
   /**
     Set a file of a given type as a member of this file set.
     If another file of the given type already exists, it is replaced
     with this file. If no member exists, a new entry is created and
-    automaticall saved to the database when {@link DbControl#commit()}
+    automatically saved to the database when {@link DbControl#commit()}
     is called.
     @param file The file to set as a member
     @param type The type of the file
+    @return The new FileSetMember item
     @throws InvalidUseOfNullException If file or type is null
     @throws PermissionDeniedException If the logged in user doesn't
@@ -268 +281 @@
       file
   */
-  public void setMember(File file, DataFileType type)
+  public FileSetMember setMember(File file, DataFileType type)
   {
     checkPermission(Permission.WRITE);
@@ -276 +289 @@
     
     Map<DataFileTypeData, FileSetMemberData> members = getData().getMembers();
-    FileSetMemberData member = members.get(type.getData());
-    if (member == null)
-    {
-      FileSetMember newMember = FileSetMember.getNew(getDbControl(), this, file, type);
-      members.put(type.getData(), newMember.getData());
-      getDbControl().saveItemIf(this, newMember, false);
+    FileSetMemberData memberData = members.get(type.getData());
+    FileSetMember member = null;
+    if (memberData == null)
+    {
+      member = FileSetMember.getNew(getDbControl(), this, file, type);
+      members.put(type.getData(), member.getData());
+      getDbControl().saveItemIf(this, member, false);
     }
     else
     {
-      if (!file.getData().equals(member.getFile()))
-      {
-        member.setFile(file.getData());
-        member.setValid(null);
-      }
-    }
+      if (!file.getData().equals(memberData.getFile()))
+      {
+        memberData.setFile(file.getData());
+        memberData.setValid(null);
+      }
+      member = getDbControl().getItem(FileSetMember.class, memberData);
+    }
+    return member;
   }
   

branches/filedb/src/core/net/sf/basedb/core/FileSetMember.java

@@ -195 +195 @@
   }
   
+  public void setValid(Boolean valid, String errorMessage)
+  {
+    checkPermission(Permission.WRITE);
+    getData().setValid(valid);
+    getData().setErrorMessage(errorMessage);
+  }
+  
   /**
     Get the error message if the validation failed.

branches/filedb/src/core/net/sf/basedb/core/Install.java

@@ -107 +107 @@
     method.
   */
-  public static final int NEW_SCHEMA_VERSION = Integer.valueOf(43).intValue();
+  public static final int NEW_SCHEMA_VERSION = Integer.valueOf(44).intValue();
   
   public static synchronized void createTables(boolean update, final ProgressReporter progress)

branches/filedb/src/core/net/sf/basedb/core/RawBioAssay.java

@@ -349 +349 @@
   // -------------------------------------------
 
-  private void setPlatform(Platform platform)
-  {
+  public void setPlatform(Platform platform)
+  {
+    checkPermission(Permission.WRITE);
     if (platform == null) throw new InvalidUseOfNullException("platform");
     getData().setPlatform(platform.getData());
-  }
-  
-  private void setVariant(PlatformVariant variant)
-  {
-    if (variant == null) throw new InvalidUseOfNullException("variant");
-    getData().setVariant(variant.getData());
-  }
+    getData().setVariant(null);
+  }
+  
+  public void setVariant(PlatformVariant variant)
+  {
+    checkPermission(Permission.WRITE);
+    if (variant != null)
+    {
+      getData().setVariant(variant.getData());
+      getData().setPlatform(variant.getData().getPlatform());
+    }
+    else
+    {
+      getData().setVariant(null);
+    }
+  } 
   
   /**

branches/filedb/src/core/net/sf/basedb/core/Update.java

@@ -29 +29 @@
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
@@ -39 +40 @@
 import org.hibernate.mapping.Table;
 
-import affymetrix.fusion.cdf.FusionCDFData;
-
-import net.sf.basedb.core.data.AnyToAnyData;
 import net.sf.basedb.core.data.ArrayDesignData;
 import net.sf.basedb.core.data.DataCubeData;
@@ -53 +51 @@
 import net.sf.basedb.core.data.PluginConfigurationData;
 import net.sf.basedb.core.data.PluginDefinitionData;
+import net.sf.basedb.core.data.RawBioAssayData;
 import net.sf.basedb.core.data.SchemaVersionData;
 import net.sf.basedb.core.data.TransformationData;
+import net.sf.basedb.core.data.keyring.PluginKeys;
 import net.sf.basedb.core.dbengine.DbEngine;
 import net.sf.basedb.core.dbengine.TableInfo;
@@ -491 +491 @@
     </td>
   </tr>
-
+  
   <tr>
     <td>42</td>
     <td>
-      <ul>
-      <li>Added {@link net.sf.basedb.core.data.ArrayDesignData#getNumFeatures()}
-      </ul>
-      The update will query the database or check the CDF file to get
-      the information.
-    </td>
-  </tr>
-  
+      No longer used.
+    </td>
+  </tr>
+
   <tr>
     <td>43</td>
@@ -511 +507 @@
     </td>
   </tr>
+
+  <tr>
+    <td>44</td>
+    <td>
+      <ul>
+      <li>Added {@link Platform}, {@link DataFileType} and
+        a lot of related classes.
+      </ul>
+      The update will set a platform for all array designs and
+      raw bioassays. For Affymetrix data it will move the CEL and CDF files
+      from AnyToAny links into FileSet:s.
+    </td>
+  </tr>
+
 
   </table>
@@ -726 +736 @@
       }
 
-      /*
-      if (schemaVersion < 43)
-      {
-        if (progress != null) progress.display((int)(42*progress_factor), "--Updating schema version: " + schemaVersion + " -> 43...");
-        schemaVersion = setSchemaVersionInTransaction(session, 43);
-        - or - 
-        schemaVersion = updateToSchemaVersion43(session);
-      }
-      ... etc... 
-      */
+      if (schemaVersion < 44)
+      {
+        if (progress != null) progress.display((int)(43*progress_factor), "--Updating schema version: " + schemaVersion + " -> 44...");
+        schemaVersion = updateToSchemaVersion44(sc);
+      }
 
       sc.logout();
@@ -1605 +1610 @@
 
   /**
-    Count the number of features for all array designs. For Affymetrix designs
-    the CDF file is checked.
+    Has been replaced with {@link #updateToSchemaVersion44(SessionControl)}
     @return The new schema version (=42)
   */
@@ -1612 +1616 @@
   {
     final int schemaVersion = 42;
-    org.hibernate.Transaction tx = null;
-    try
-    {
-      tx = HibernateUtil.newTransaction(session);
-  
-      org.hibernate.Query query = HibernateUtil.createQuery(session, 
-        "SELECT ad FROM ArrayDesignData ad");
-      org.hibernate.Query countQuery = HibernateUtil.getPredefinedQuery(session, 
-        "COUNT_FEATURES_FOR_ARRAYDESIGN");
-      org.hibernate.Query cdfQuery = HibernateUtil.getPredefinedQuery(session, 
-        "GET_ANYTOANY_FOR_NAME");
-      cdfQuery.setInteger("fromType", Item.ARRAYDESIGN.getValue());
-      cdfQuery.setString("name", Affymetrix.CDF_LINK_NAME);
-      
-      List<ArrayDesignData> designs = HibernateUtil.loadList(ArrayDesignData.class, query);
-      for (ArrayDesignData design : designs)
-      {
-        int numFeatures = -1;
-        try
-        {
-          if (!design.getHasFeatures())
-          {
-            numFeatures = 0;
-          }
-          else  if (design.isAffyChip())
-          {
-            // Get CDF file and the number of features
-            cdfQuery.setInteger("fromId", design.getId());
-            AnyToAnyData cdfLink = HibernateUtil.loadData(AnyToAnyData.class, cdfQuery);
-            FileData cdf = null;
-            if (cdfLink != null)
-            {
-              cdf = HibernateUtil.loadData(session, FileData.class, cdfLink.getToId());
-            }
-            if (cdf != null)
-            {
-              FusionCDFData cdfData = new FusionCDFData();
-              cdfData.setFileName(File.getAbsolutePath(cdf.getInternalName()).getPath());
-              cdfData.readHeader();
-              numFeatures = cdfData.getHeader().getNumProbeSets();
-            }
-            else
-            {
-              numFeatures = 0;
-            }
-          }
-          else
-          {
-            countQuery.setInteger("arrayDesign", design.getId());
-            numFeatures = HibernateUtil.loadData(Long.class, countQuery).intValue();
-          }
-        }
-        catch (Throwable t)
-        {
-          log.error("Could not set number of features on array design: " + design, t);
-        }
-        //design.setNumFeatures(numFeatures);
-      }
-
-  
-      // Update the schema version number
-      setSchemaVersion(session, schemaVersion);
-  
-      // Commit the changes
-      HibernateUtil.commit(tx);
-      log.info("updateToSchemaVersion42: OK");
-    }
-    catch (BaseException ex)
-    {
-      if (tx != null) HibernateUtil.rollback(tx); 
-      log.error("updateToSchemaVersion42: FAILED", ex);
-      throw ex;
-    }
     return schemaVersion;   
   }
@@ -1728 +1659 @@
   }
 
+  /**
+    Set a platform for array designs and raw bioassays. 
+    Move CEL and CDF files into FileSet:s
+    Remove filters on isAffyChip property
+  */
+  @SuppressWarnings("deprecation")
+  private static int updateToSchemaVersion44(SessionControl sc)
+  {
+    final int schemaVersion = 44;
+    DbControl dc = null;
+    try
+    {
+      dc = sc.newDbControl();
+      org.hibernate.Session session = dc.getHibernateSession();
+  
+      // Items that we need
+      Platform affymetrix = Platform.getByExternalId(dc, Platform.AFFYMETRIX);
+      Platform generic = Platform.getByExternalId(dc, Platform.GENERIC);
+      DataFileType cdfType = DataFileType.getByExternalId(dc, DataFileType.AFFYMETRIX_CDF);
+      DataFileType celType = DataFileType.getByExternalId(dc, DataFileType.AFFYMETRIX_CEL);
+      
+      // Query to count number of features in db
+      org.hibernate.Query countQuery = HibernateUtil.getPredefinedQuery(session, 
+        "COUNT_FEATURES_FOR_ARRAYDESIGN");
+
+      // Load array designs
+      org.hibernate.Query designQuery = HibernateUtil.createQuery(session, 
+        "SELECT ad FROM ArrayDesignData ad WHERE ad.platform is null");
+      List<ArrayDesignData> arrayDesigns = 
+        HibernateUtil.loadList(ArrayDesignData.class, designQuery);
+      for (ArrayDesignData dd : arrayDesigns)
+      {
+        ArrayDesign design = dc.getItem(ArrayDesign.class, dd);
+        if (design.getData().isAffyChip())
+        {
+          design.setPlatform(affymetrix);
+          AnyToAny anyCdf = null;
+          try
+          {
+            anyCdf = AnyToAny.getByName(dc, design, Affymetrix.CDF_LINK_NAME);
+          }
+          catch (Throwable t)
+          {}
+          AnyToAny.unlinkFrom(dc, design, Affymetrix.CDF_LINK_NAME);
+          if (anyCdf != null)
+          {
+            File cdfFile = (File)anyCdf.getTo();
+            FileSet fileSet = design.getFileSet();
+            fileSet.setMember(cdfFile, cdfType);
+            fileSet.validate(dc, true);
+          }
+        }
+        else
+        {
+          design.setPlatform(generic);
+          countQuery.setInteger("arrayDesign", design.getId());
+          int dbFeatures = HibernateUtil.loadData(Long.class, countQuery).intValue();
+          design.getData().setNumDbFeatures(dbFeatures);
+        }
+      }
+      
+      // Load rawbioassays
+      org.hibernate.Query rbaQuery = HibernateUtil.createQuery(session, 
+        "SELECT rba FROM RawBioAssayData rba WHERE rba.platform is null");
+      List<RawBioAssayData> rawBioAssays = 
+        HibernateUtil.loadList(RawBioAssayData.class, rbaQuery);
+      for (RawBioAssayData dd : rawBioAssays)
+      {
+        RawBioAssay rba = dc.getItem(RawBioAssay.class, dd);
+        if ("affymetrix".equals(rba.getData().getRawDataType()))
+        {
+          rba.setPlatform(affymetrix);
+          AnyToAny anyCel = null;
+          try
+          {
+            anyCel = AnyToAny.getByName(dc, rba, Affymetrix.CEL_LINK_NAME);
+          }
+          catch (Throwable t)
+          {}
+          AnyToAny.unlinkFrom(dc, rba, Affymetrix.CEL_LINK_NAME);
+          if (anyCel != null)
+          {
+            File celFile = (File)anyCel.getTo();
+            FileSet fileSet = rba.getFileSet();
+            fileSet.setMember(celFile, cdfType);
+            fileSet.validate(dc, true);
+          }
+        }
+        else
+        {
+          rba.setPlatform(generic);
+        }
+      }
+  
+      // Remove filters on the 'affyChip' property
+      org.hibernate.Query query = HibernateUtil.getPredefinedSQLQuery(session, 
+        "DELETE_PROPERTY_FILTER");
+      /*
+        DELETE FROM PropertyFilters pf
+        WHERE pf.property = :property
+       */
+      query.setString("property", "affyChip");
+      HibernateUtil.executeUpdate(query);
+
+      // Add READ permission to PLATFORM and DATAFILETYPE for all plug-ins
+      // working with ARRAYDESIGN or RAWBIOASSAY
+      int adKeyId = SystemItems.getRoleKeyId(Item.ARRAYDESIGN);
+      int rbaKeyId = SystemItems.getRoleKeyId(Item.RAWBIOASSAY);
+      
+      org.hibernate.Query pQuery = HibernateUtil.getPredefinedQuery(session, 
+        "GET_PLUGINS_WITH_KEYS");
+      /*
+        SELECT DISTINCT pk.pluginDefinitionId
+        FROM PluginKeys pk WHERE pk.keyId IN (:keys)
+      */
+      pQuery.setParameterList("keys", 
+        Arrays.asList( new Integer[] { adKeyId, rbaKeyId }), Type.INT.getHibernateType());
+      List<Integer> pluginIds = HibernateUtil.loadList(Integer.class, pQuery);
+      for (int pluginId : pluginIds)
+      {
+        addPluginPermission(session, pluginId, Item.PLATFORM, 0, 2046);
+        addPluginPermission(session, pluginId, Item.DATAFILETYPE, 0, 2046);
+      }
+      
+      // Update the schema version number
+      setSchemaVersion(session, schemaVersion);
+  
+      // Commit the changes
+      dc.commit();
+      log.info("updateToSchemaVersion44: OK");
+    }
+    catch (BaseException ex)
+    {
+      log.error("updateToSchemaVersion44: FAILED", ex);
+      throw ex;
+    }
+    finally
+    {
+      if (dc != null) dc.close();
+    }
+    return schemaVersion;   
+  }
+
+  private static void addPluginPermission(org.hibernate.Session session, int pluginId, Item itemType, int granted, int denied)
+  {
+    PluginKeys pk = new PluginKeys();
+    pk.setPluginDefinitionId(pluginId);
+    pk.setKeyId(SystemItems.getRoleKeyId(itemType));
+    if (session.get(PluginKeys.class, pk) == null)
+    {
+      pk.setGranted(granted);
+      pk.setDenied(denied);
+      session.save(pk);
+    }
+  }
+  
   /**
     Adjust the existing items in the database to be compatible with the latest mappings.

branches/filedb/src/plugins/core/net/sf/basedb/plugins/CdfFileReporterImporter.java

@@ -45 +45 @@
 import net.sf.basedb.core.ArrayDesign;
 import net.sf.basedb.core.BaseException;
+import net.sf.basedb.core.DataFileType;
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.File;
 import net.sf.basedb.core.FileParameterType;
+import net.sf.basedb.core.FileStoreUtil;
 import net.sf.basedb.core.InvalidDataException;
 import net.sf.basedb.core.Item;
@@ -57 +59 @@
 import net.sf.basedb.core.Operator;
 import net.sf.basedb.core.Permission;
+import net.sf.basedb.core.Platform;
 import net.sf.basedb.core.PluginParameter;
 import net.sf.basedb.core.ProgressReporter;
@@ -65 +68 @@
 import net.sf.basedb.core.StringParameterType;
 import net.sf.basedb.core.Type;
+import net.sf.basedb.core.filehandler.CdfFileHandler;
 import net.sf.basedb.core.plugin.About;
 import net.sf.basedb.core.plugin.AboutImpl;
@@ -156 +160 @@
   /**
     Request create and write access to Reporter:s
-    and read access to File:s ArrayDesign:s and ReporterType:s.
+    and read access to File:s ArrayDesign:s, ReporterType:s,
+    Platform:s and DataFileType:s
   */
   public Collection<Permissions> getPermissions()
@@ -166 +171 @@
       permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
       permissions.add(new Permissions(Item.ARRAYDESIGN, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.PLATFORM, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.DATAFILETYPE, null, EnumSet.of(Permission.READ)));
     }
     return permissions;
@@ -178 +185 @@
       File cdfFile = (File)job.getValue("file");
       cdfFile = File.getById(dc, cdfFile.getId());
-      FusionCDFData cdf = Affymetrix.loadCdfFile(cdfFile);
+      FusionCDFData cdf = new CdfFileHandler().loadCdfFile(cdfFile);
   
       // Import data from it
@@ -225 +232 @@
       }
       ArrayDesign design = (ArrayDesign)item;
-      if (!design.isAffyChip())
+      if (!design.isPlatform(Platform.AFFYMETRIX))
       {
         message = "The array design is not an Affymetrix chip";
@@ -231 +238 @@
       else
       {
-        File cdfFile = Affymetrix.getCdfFile(design);
-        if (cdfFile == null) 
-        {
-          throw new InvalidDataException("The array design doesn't have a CDF file");
+        if (!FileStoreUtil.hasDataFile(design.getDbControl(), design, DataFileType.AFFYMETRIX_CDF, true))
+        {
+          throw new InvalidDataException("The array design doesn't have a valid CDF file");
         }
       }
@@ -274 +280 @@
         // Verify that the file is a CDF file
         File cdfFile = (File)request.getParameterValue("file");
-        FusionCDFData cdf = Affymetrix.loadCdfFile(cdfFile);
+        FusionCDFData cdf = new CdfFileHandler().loadCdfFile(cdfFile);
         
         storeValue(job, request, ri.getParameter("file"));
@@ -383 +389 @@
           int currentArrayDesignId = sc.getCurrentContext(Item.ARRAYDESIGN).getId();
           design = ArrayDesign.getById(dc, currentArrayDesignId);
-          cdfFile = Affymetrix.getCdfFile(design);
+          cdfFile = FileStoreUtil.getDataFile(dc, design, DataFileType.AFFYMETRIX_CDF);
           presets = Arrays.asList(new File[] { cdfFile } );
         }

branches/filedb/src/plugins/core/net/sf/basedb/plugins/PrintMapFlatFileImporter.java

@@ -32 +32 @@
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.FeatureBatcher;
+import net.sf.basedb.core.File;
+import net.sf.basedb.core.FileParameterType;
+import net.sf.basedb.core.FileSetMember;
+import net.sf.basedb.core.FileStoreUtil;
 import net.sf.basedb.core.FileType;
 import net.sf.basedb.core.IntegerUtil;
@@ -122 +126 @@
 
   private ArrayDesign arrayDesign;
+  private FileSetMember printMapMember;
   private ItemParameterType<ArrayDesign> arrayDesignType;
   private PluginParameter<ArrayDesign> arrayDesignParameter;
@@ -150 +155 @@
   }
   /**
-    Request write access to ArrayDesign:s, read access to Plate:s, Reporter:s
-    and File:s.
+    Request write access to ArrayDesign:s, read access to Plate:s, Reporter:s,
+    DataFileType:s, Platform:s and File:s.
   */
   public Collection<Permissions> getPermissions()
@@ -161 +166 @@
       permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.READ)));
       permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.PLATFORM, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.DATAFILETYPE, null, EnumSet.of(Permission.READ)));
     }
     return permissions;
@@ -188 +195 @@
     {
       ArrayDesign ad = (ArrayDesign)item;
-      if (ad.isAffyChip())
-      {
-        message = "Affy array designs are not supported";
-      }
-      else if (ad.hasFeatures())
+      if (ad.isFileOnlyPlatform())
+      {
+        message = "Array design is using a file-only platform";
+      }
+      else if (ad.getNumDbFeatures() > 0)
       {
         throw new PermissionDeniedException("The array design already has features");
@@ -301 +308 @@
     arrayDesign = (ArrayDesign)job.getValue("arrayDesign");
     arrayDesign = ArrayDesign.getById(dc, arrayDesign.getId());
-    if (arrayDesign.hasFeatures())
+    File reporterMapFile = (File)job.getValue("file");
+    printMapMember = 
+      FileStoreUtil.setDataFile(dc, arrayDesign, "generic.printmap", reporterMapFile);
+    if (arrayDesign.getNumDbFeatures() > 0)
     {
       throw new PermissionDeniedException("The array design already has features: " + arrayDesign.getName()+" ["+arrayDesign.getId()+"]");
@@ -447 +457 @@
     throws BaseException
   {
+    if (printMapMember != null) printMapMember.setValid(success, null);
     if (blocks != null) blocks.clear();
     try
@@ -503 +514 @@
       List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
       parameters.add(arrayDesignParameter);
+
+      ArrayDesign design = null;
+      int currentArrayDesignId = sc.getCurrentContext(Item.ARRAYDESIGN).getId();
+      List<File> printMapFiles = null;
+      dc = sc.newDbControl();
+      try
+      {
+        design = ArrayDesign.getById(dc, currentArrayDesignId);
+        printMapFiles = FileStoreUtil.getGenericDataFiles(dc, design, FileType.PRINT_MAP);
+      }
+      catch (Throwable t)
+      {}
+      finally
+      {
+        if (dc != null) dc.close();
+      }
+      
+      // The print map file to import from - if a file already hase
+      // been attached to the array design use it as a default choice
+      PluginParameter<File> fileParameter = new PluginParameter<File>(
+        "file",
+        "Print map file",
+        "The file with printing information to import features from",
+        new FileParameterType(printMapFiles == null || printMapFiles.isEmpty() ? 
+          null : printMapFiles.get(0), true, 1)
+      );
       parameters.add(fileParameter);
       parameters.add(getCharsetParameter(null, null, null));

branches/filedb/src/plugins/core/net/sf/basedb/plugins/ReporterMapFlatFileImporter.java

@@ -32 +32 @@
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.FeatureBatcher;
+import net.sf.basedb.core.File;
+import net.sf.basedb.core.FileParameterType;
+import net.sf.basedb.core.FileSetMember;
+import net.sf.basedb.core.FileStoreUtil;
 import net.sf.basedb.core.FileType;
 import net.sf.basedb.core.InvalidUseOfNullException;
@@ -173 +177 @@
   private FeatureBatcher batcher;
   private ReporterBatcher reporterBatcher;
+  private FileSetMember reporterMapMember;
   private ArrayDesign arrayDesign;
   private Map<BlockInfo, ArrayDesignBlock> blocks;
@@ -216 +221 @@
   }
   /**
-    Request write access to ArrayDesign:s, read access to Reporter:s
-    and File:s.
+    Request write access to ArrayDesign:s, read access to Reporter:s,
+    DataFileType:s, Platform:s and File:s.
   */
   public Collection<Permissions> getPermissions()
@@ -226 +231 @@
       permissions.add(new Permissions(Item.REPORTER, null, EnumSet.of(Permission.READ)));
       permissions.add(new Permissions(Item.FILE, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.PLATFORM, null, EnumSet.of(Permission.READ)));
+      permissions.add(new Permissions(Item.DATAFILETYPE, null, EnumSet.of(Permission.READ)));
     }
     return permissions;
@@ -263 +270 @@
     {
       ArrayDesign ad = (ArrayDesign)item;
-      if (ad.isAffyChip())
-      {
-        message = "Affy array designs are not supported";
-      }
-      else if (ad.hasFeatures())
+      if (ad.isFileOnlyPlatform())
+      {
+        message = "Array design is using a file-only platform";
+      }
+      else if (ad.getNumDbFeatures() > 0)
       {
         throw new PermissionDeniedException("The array design already has features");
@@ -398 +405 @@
     dc = sc.newDbControl();
     arrayDesign = (ArrayDesign)job.getValue("arrayDesign");
-    dc.reattachItem(arrayDesign);
-    if (arrayDesign.hasFeatures())
+    File reporterMapFile = (File)job.getValue("file");
+    arrayDesign = ArrayDesign.getById(dc, arrayDesign.getId());
+    reporterMapMember = 
+      FileStoreUtil.setDataFile(dc, arrayDesign, "generic.reportermap", reporterMapFile);
+    if (arrayDesign.getNumDbFeatures() > 0)
     {
       throw new PermissionDeniedException("The array design already has features. "+arrayDesign.getName()+"["+arrayDesign.getId()+"]");
@@ -486 +496 @@
   {
     if (blocks != null) blocks.clear();
+    if (reporterMapMember != null) reporterMapMember.setValid(success, null);
     try
     {
@@ -542 +553 @@
       List<PluginParameter<?>> parameters = new ArrayList<PluginParameter<?>>();
       parameters.add(arrayDesignParameter);
+
+      ArrayDesign design = null;
+      int currentArrayDesignId = sc.getCurrentContext(Item.ARRAYDESIGN).getId();
+      List<File> reporterMapFiles = null;
+      dc = sc.newDbControl();
+      try
+      {
+        design = ArrayDesign.getById(dc, currentArrayDesignId);
+        reporterMapFiles = FileStoreUtil.getGenericDataFiles(dc, design, FileType.REPORTER_MAP);
+      }
+      catch (Throwable t)
+      {}
+      finally
+      {
+        if (dc != null) dc.close();
+      }
+      
+      // The reporter map file to import from - if a file already hase
+      // been attached to the array design use it as a default choice
+      PluginParameter<File> fileParameter = new PluginParameter<File>(
+        "file",
+        "Reporter map file",
+        "The file with reporter information to import features from",
+        new FileParameterType(reporterMapFiles == null || reporterMapFiles.isEmpty() ? 
+          null : reporterMapFiles.get(0), true, 1)
+      );
       parameters.add(fileParameter);
       parameters.add(getCharsetParameter(null, null, (String)configuration.getValue(CHARSET)));

branches/filedb/src/test/TestArrayDesign.java

@@ -43 +43 @@
 import net.sf.basedb.core.ReporterBatcher;
 import net.sf.basedb.core.ItemResultList;
-import net.sf.basedb.core.SystemItems;
 import net.sf.basedb.core.Well;
 import net.sf.basedb.core.data.FeatureData;
@@ -105 +104 @@
     // Array designs with data in files
     int cdfId = TestFile.test_create("data/test.affymetrix.cdf", false, false);
-    test_set_file(id3, "affymetrix.cdf", cdfId);
+    test_set_file(id3, DataFileType.AFFYMETRIX_CDF, cdfId);
     test_list_files(id3, 1);
 
@@ -119 +118 @@
     TestPlateType.test_delete(plateType);
     TestPlateGeometry.test_delete(plateGeometry);
-    
+    TestFile.test_delete(cdfId);
     TestReporter.test_delete();
 
@@ -604 +603 @@
       dc = TestUtil.getDbControl();
       ArrayDesign ad = ArrayDesign.getById(dc, arrayDesignId);
-      DataFileType type = DataFileType.getById(dc, SystemItems.getId(fileType));
+      DataFileType type = DataFileType.getByExternalId(dc, fileType);
       File file = File.getById(dc, fileId);
       ad.getFileSet().setMember(file, type);
+      ad.getFileSet().validate(dc, true);
       dc.commit();
       write("--Set file OK ("+fileType+")");

branches/filedb/www/common/datafiles/list_files.jsp

@@ -184 +184 @@
           {
             FileSetMember member = entry.getValue();
+            DataFileType dft = entry.getKey();
             Boolean validFile = null;
             String validationMessage = "";
@@ -189 +190 @@
             File file = null;
             boolean readDataFileType = true;
-            DataFileType dft = null;
             if (member != null)
             {