Changeset 3814

Timestamp:

Oct 10, 2007, 2:57:07 PM (16 years ago)

Author:

Nicklas Nordborg

Message:

References #721: Store data in files instead of in the database

• Experiments are now aware of file-only raw data types
• Fixed test programs
• A few remaining Affymetrix issues

Location:

branches/filedb

Files:

• src/core/common-queries.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/Affymetrix.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/RawDataType.java (modified) (6 diffs)
• src/core/net/sf/basedb/core/RawDataTypes.java (modified) (3 diffs)
• src/core/net/sf/basedb/core/Update.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/filehandler/AffymetrixFileHandler.java (modified) (2 diffs)
• src/test/TestDirty.java (modified) (1 diff)
• src/test/data/test.affymetrix.cdf (added)
• src/test/data/test.affymetrix.cel (added)
• www/views/experiments/edit_experiment.jsp (modified) (4 diffs)
• www/views/experiments/list_experiments.jsp (modified) (1 diff)
• www/views/rawbioassays/list_rawbioassays.jsp (modified) (2 diffs)

branches/filedb/src/core/common-queries.xml

@@ -2895 +2895 @@
   </query>
 
+  <query id="CHANGE_EXPERIMENT_RAWDATATYPE" type="HQL">
+    <sql>
+      UPDATE ExperimentData
+      SET rawDataType = :newRawDataType
+      WHERE rawDataType = :oldRawDataType
+    </sql>
+    <description>
+      A HQL query that changes all raw data type for experiments from
+      an old value to a new value.
+    </description>
+  </query>
+
 
 </predefined-queries>

branches/filedb/src/core/net/sf/basedb/core/Affymetrix.java

@@ -200 +200 @@
   {
     if (rawBioAssay == null) throw new InvalidUseOfNullException("rawBioAssay");
-    if (!rawBioAssay.getRawDataType().isAffymetrix()) 
+    if (!rawBioAssay.isPlatform(Platform.AFFYMETRIX)) 
     {
       throw new InvalidDataException("Raw data type for " + rawBioAssay.getName() + 
-      " is not an Affymetrix raw data type");
+        " is not an Affymetrix raw data type");
     }
     DbControl dc = rawBioAssay.getDbControl();
-    File celFile = null;
-    try
-    {
-      AnyToAny ata = AnyToAny.getByName(dc, rawBioAssay, CEL_LINK_NAME);
-      if (ata.getToType() == Item.FILE)
-      {
-        celFile = (File)ata.getTo();
-      }
-    }
-    catch (ItemNotFoundException ex)
-    {}
-    return celFile;
+    return FileStoreUtil.getDataFile(dc, rawBioAssay, DataFileType.AFFYMETRIX_CEL);
   }
   
@@ -234 +223 @@
   public static void validateCelAndCdf(RawBioAssay rawBioAssay, ArrayDesign design, boolean required)
   {
-    File celFile = getCelFile(rawBioAssay);
-    File cdfFile = getCdfFile(design);
+    DbControl dc = rawBioAssay.getDbControl();
+    File celFile = FileStoreUtil.getDataFile(dc, rawBioAssay, DataFileType.AFFYMETRIX_CEL);;
+    File cdfFile = FileStoreUtil.getDataFile(dc, design, DataFileType.AFFYMETRIX_CDF);
     if (celFile != null && cdfFile != null)
     {
       String cdfChipType = cdfFile.getName();
-      FusionCELData cel = loadCelFile(celFile);
-      FusionCDFData cdf = loadCdfFile(cdfFile);
+      FusionCELData cel = new CelFileHandler().loadCelFile(celFile);
+      FusionCDFData cdf = new CdfFileHandler().loadCdfFile(cdfFile);
       validateCelAndCdf(cel, cdf, cdfChipType);
     }

branches/filedb/src/core/net/sf/basedb/core/RawDataType.java

@@ -56 +56 @@
   private final boolean isAffymetrix;
   private final int channels;
+  private final int platformId;
+  private final int variantId;
   private String table;
   private RealTable realTable;
@@ -71 +73 @@
     String table, List<RawDataProperty> properties, List<IntensityFormula> formulas)
   {
-    this(id, name, description, channels, false, false, table, properties, formulas);
-  }
-  
+    this(id, name, description, channels, false, 0, 0, false, table, properties, formulas);
+  }
+  
+  /**
+    Create a new raw data type representing a file-only platform.
+    @param platform The platform
+  */
   RawDataType(PlatformData platform)
   {
     this("platform." + platform.getExternalId(), platform.getName(), 
       platform.getDescription(), platform.getChannels(), true, 
+      platform.getId(), 0,
       Platform.AFFYMETRIX.equals(platform.getExternalId()),
       null, null, null);
   }
 
+  /**
+    Create a new raw data type representing a file-only platform variant.
+    @param variant The platform variant
+  */
   RawDataType(PlatformVariantData variant)
   {
     this("variant." + variant.getExternalId(), variant.getName(), 
         variant.getDescription(), variant.getChannels(), true, 
+        variant.getPlatform().getId(), variant.getId(),
         Platform.AFFYMETRIX.equals(variant.getPlatform().getExternalId()),
         null, null, null);    
@@ -91 +103 @@
   
   private RawDataType(String id, String name, String description, int channels, 
-    boolean fileOnly, boolean isAffymetrix,
+    boolean fileOnly, int platformId, int variantId, boolean isAffymetrix,
     String table, List<RawDataProperty> properties, List<IntensityFormula> formulas)
   {
@@ -99 +111 @@
     this.channels = channels;
     this.fileOnly = fileOnly;
+    this.platformId = platformId;
+    this.variantId = variantId;
     this.isAffymetrix = isAffymetrix;
     
@@ -158 +172 @@
   /**
     If this raw data type is the Affymetrix raw data type.
-    @deprecated Use --- instead
+    @deprecated Use {@link #getPlatform(DbControl)} and compare
+      the external ID with {@link Platform#AFFYMETRIX}
   */
   public boolean isAffymetrix()
@@ -165 +180 @@
   }
   
+  public Platform getPlatform(DbControl dc)
+  {
+    return platformId == 0 ? null : Platform.getById(dc, platformId);
+  }
+
+  public PlatformVariant getVariant(DbControl dc)
+  {
+    return variantId == 0 ? null : PlatformVariant.getById(dc, variantId);
+  }
+
   /**
     Get the name of this raw data type. This value is the same as the

branches/filedb/src/core/net/sf/basedb/core/RawDataTypes.java

@@ -161 +161 @@
 
   /**
-    Get a collection with all raw data types that has been defined.
+    Get a collection with all raw data types that has been defined. This method
+    only returns raw data types that are importable to the database. To
+    get file-only raw data types use {@link #getFileOnlyRawDataTypes()}
     @return A <code>Collection</code> containing {@link RawDataType} objects
+      where {@link RawDataType#isStoredInDb()} returns true
   */
   public static Collection<RawDataType> getRawDataTypes()
   {
     return Collections.unmodifiableCollection(rawDataTypes.values());
+  }
+  
+  /**
+    Get the raw data types that are representations of file-only
+    {@link Platform}:s.
+    @return A <code>Collection</code> containing {@link RawDataType} objects 
+      where {@link RawDataType#isStoredInDb()} returns false
+    @since 2.5
+  */
+  public static Collection<RawDataType> getFileOnlyRawDataTypes()
+  {
+    return Collections.unmodifiableCollection(platformTypes.values());
   }
 
@@ -203 +218 @@
     try
     {
+      session = HibernateUtil.newSession();
+      tx = HibernateUtil.newTransaction(session);
       org.hibernate.Query query = HibernateUtil.getPredefinedQuery(session, 
         "GET_PLATFORM_FOR_EXTERNAL_ID");
@@ -228 +245 @@
     try
     {
+      session = HibernateUtil.newSession();
+      tx = HibernateUtil.newTransaction(session);
       org.hibernate.Query query = HibernateUtil.getPredefinedQuery(session, 
         "GET_PLATFORMVARIANT_FOR_EXTERNAL_ID");

branches/filedb/src/core/net/sf/basedb/core/Update.java

@@ -1660 +1660 @@
 
   /**
-    Set a platform for array designs and raw bioassays. 
-    Move CEL and CDF files into FileSet:s
-    Remove filters on isAffyChip property
+    <ul>
+    <li>Set a platform for array designs and raw bioassays
+    <li>Change experiment raw data type if it is "affymetrix" to "platform.affymetrix"
+    <li>Move CEL and CDF files into FileSet:s
+    <li>Remove filters on ArrayDesign.isAffyChip property
+    <li>Rename filter on RawBioassay.spots property to RawBioassay.numDbSpots
+    <li>Add READ permission to PLATFORM and DATAFILETYPE for all plug-ins
+      working with ARRAYDESIGN or RAWBIOASSAY
+    </ul>
   */
   @SuppressWarnings("deprecation")
@@ -1753 +1759 @@
       }
   
+      // Change raw data type for 'affymetrix' experiments to 'platform.affymetrix'
+      org.hibernate.Query query = HibernateUtil.getPredefinedSQLQuery(session, 
+        "CHANGE_EXPERIMENT_RAWDATATYPE");
+      /*
+        UPDATE ExperimentData
+        SET rawDataType = :newRawDataType
+        WHERE rawDataType = :oldRawDataType
+      */
+      query.setString("oldRawDataType", "affymetrix");
+      query.setString("newRawDataType", "platform.affymetrix");
+      HibernateUtil.executeUpdate(query);
+      
       // Remove filters on the 'affyChip' property
-      org.hibernate.Query query = HibernateUtil.getPredefinedSQLQuery(session, 
+      query = HibernateUtil.getPredefinedSQLQuery(session, 
         "DELETE_PROPERTY_FILTER");
       /*

branches/filedb/src/core/net/sf/basedb/core/filehandler/AffymetrixFileHandler.java

@@ -70 +70 @@
     @param cdfFile The file to load
     @return A FusionCDFData object representing the CDF file
-    @throws ItemNotFoundException If the actaul file is not on the server
+    @throws ItemNotFoundException If the actual file is not on the server
     @throws InvalidDataException If the file is not a CDF file
   */
@@ -93 +93 @@
   }
   
+  /**
+    Load a CEL file using the Affymetric Fusion SDK. This method
+    checks that the file exists, and read all headers.
+    
+    @param celFile The file to load
+    @return A FusionCELData object representing the CEL file
+    @throws ItemNotFoundException If the actual file is not on the server
+    @throws InvalidDataException If the file is not a CEL file
+  */
   public FusionCELData loadCelFile(File celFile)
   {

branches/filedb/src/test/TestDirty.java

@@ -92 +92 @@
     TestJob.test_delete(jobId);
     TestPluginConfiguration.test_delete(pluginConfigurationId);
-    TestFile.test_delete(fileId);
     TestRawBioAssay.test_delete(rawBioAssayId);
     TestArrayDesign.test_delete(arrayDesignId);
+    TestFile.test_delete(fileId);
     TestReporter.test_delete();
     write("++Testing import of dirty data "+(ok ? "OK" : "Failed")+"\n");

branches/filedb/www/views/experiments/edit_experiment.jsp

@@ -54 +54 @@
   import="net.sf.basedb.clients.web.util.HTML"
   import="net.sf.basedb.util.Values"
+  import="net.sf.basedb.util.Enumeration"
   import="net.sf.basedb.util.formatter.Formatter"
   import="net.sf.basedb.clients.web.formatter.FormatterFactory"
@@ -151 +152 @@
         frm.name.focus();
         return false;
+      }
+      if (frm.rawdatatype)
+      {
+        if (frm.rawdatatype[frm.rawdatatype.selectedIndex].value == '')
+        {
+          alert("You must select a raw data type");
+          frm.rawdatatype.focus();
+          return false;
+        }
       }
       return true;
@@ -334 +344 @@
           if (experiment == null)
           {
-            %>
-            <select name="rawdatatype" class="required">
-            <%
-            currentRawDataType = currentRawDataType != null ? currentRawDataType : defaultRawDataType;
+            currentRawDataType = currentRawDataType != null ? 
+              currentRawDataType : defaultRawDataType;
+            Enumeration<RawDataType, String> rawEnumeration = 
+              new Enumeration<RawDataType, String>();
             for (RawDataType rdt : RawDataTypes.getRawDataTypes())
             {
+              rawEnumeration.add(rdt, rdt.getName());
+            }
+            for (RawDataType rdt : RawDataTypes.getFileOnlyRawDataTypes())
+            {
+              rawEnumeration.add(rdt, rdt.getName());
+            }
+            rawEnumeration.sortValues();            
+            %>
+            <select name="rawdatatype" class="required unchangeable">
+            <%
+            for (int i = 0; i < rawEnumeration.size(); ++i)
+            {
+              RawDataType rdt = rawEnumeration.getKey(i);
               String selected = rdt.equals(currentRawDataType) ? "selected" : "";
               %>
@@ -413 +436 @@
       </tr>
       </table>
-      <div align=right>&nbsp;<i><base:icon image="required.gif" /> = required information</i></div>
+      <div align=right>
+        &nbsp;<i><base:icon image="required.gif" /> = required information</i>
+        <%if (experiment == null) {%><br>
+        <i><base:icon image="unchangeable.gif" /> = can't be changed later</i>
+        <%}%>
+        </div>
     </t:tab>
     

branches/filedb/www/views/experiments/list_experiments.jsp

@@ -226 +226 @@
         rawEnumeration.add(rdt.getId(), HTML.encodeTags(rdt.getName()));
       }
+      for (RawDataType rdt : RawDataTypes.getFileOnlyRawDataTypes())
+      {
+        rawEnumeration.add(rdt.getId(), HTML.encodeTags(rdt.getName()));
+      }
+      rawEnumeration.sortValues();
       %>
       <tbl:columndef 

branches/filedb/www/views/rawbioassays/list_rawbioassays.jsp

@@ -282 +282 @@
         filterable="true"
         exportable="true"
-      />        <%
+      />        
+      <%
       Enumeration<String, String> rawEnumeration = new Enumeration<String, String>();
       for (RawDataType rdt : RawDataTypes.getRawDataTypes())
@@ -288 +289 @@
         rawEnumeration.add(rdt.getId(), HTML.encodeTags(rdt.getName()));
       }
+      for (RawDataType rdt : RawDataTypes.getFileOnlyRawDataTypes())
+      {
+        rawEnumeration.add(rdt.getId(), HTML.encodeTags(rdt.getName()));
+      }
+      rawEnumeration.sortValues();
       %>
       <tbl:columndef