Changeset 3523

Timestamp:

Jun 20, 2007, 2:32:41 PM (16 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #583: Allow changing the array design of a raw bioasay when it has raw data

Location:

trunk

Files:

• doc/src/docbook/userdoc/rawbioassays.xml (modified) (1 diff)
• src/core/net/sf/basedb/core/Affymetrix.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/Job.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/RawBioAssay.java (modified) (5 diffs)
• src/core/net/sf/basedb/core/templates/hibernate-properties-RawData.xml (modified) (1 diff)
• www/views/jobs/view_job.jsp (modified) (1 diff)
• www/views/rawbioassays/edit_rawbioassay.jsp (modified) (3 diffs)
• www/views/rawbioassays/index.jsp (modified) (6 diffs)

trunk/doc/src/docbook/userdoc/rawbioassays.xml

@@ -105 +105 @@
             since some optimzations can be used when assigning 
             positions in bioassay sets.
-            The array design cannot be changed after raw data has been 
-            imported.
+            The array design can be changed after raw data has been 
+            imported, but this triggers a new validation. If the raw data
+            is stored in the database, the position, coordinates and reporter of all features
+            on the new array design must exactly match the position,
+            coordinates and reporter of the raw data. For Affymetrix data, the
+            CEL file is validated against the CDF file of the new array design.
+            If the validation fails, the array design is not changed.
           </para>
         </listitem>

trunk/src/core/net/sf/basedb/core/Affymetrix.java

@@ -201 +201 @@
       File cdfFile = getCdfFile(design);
       FusionCDFData cdf = loadCdfFile(cdfFile);
+      validateCelAndCdf(cel, cdf, cdfFile.getName());
       
-      String celChipType = cel.getChipType();
-      String cdfChipType = cdfFile.getName();
-      if (!cdfChipType.startsWith(celChipType))
-      {
-        throw new InvalidDataException("CEL chip type (" + celChipType + 
-          ") doesn't match CDF chip type (" + cdfChipType + ")");
-      }
-      
-      if (cel.getRows() != cdf.getHeader().getRows() || cel.getCols() != cdf.getHeader().getRows())
-      {
-        throw new InvalidDataException("CEL size (rows=" + cel.getRows() + ", cols=" + cel.getCols() +
-          ") doesn't match CDF size (rows=" + 
-          cdf.getHeader().getRows() + ", cols=" + cdf.getHeader().getCols()+")");
-      }
-
       file.checkPermission(Permission.USE);
       AnyToAny ata = AnyToAny.getNewOrExisting(dc, rawBioAssay, CEL_LINK_NAME, file, true);
@@ -285 +271 @@
     {}
     return celFile;
+  }
+  
+  /**
+    Check if the CEL file on the raw bioassay matches the CDF file on the
+    array design.
+    
+    @param rawBioAssay The raw bioassay
+    @param design The array design
+    @param required TRUE if the absense of either a CEL or a CDF file is an
+      error condition
+    @throws InvalidDataException If the raw bioassay or array design are not
+      Affymetrix type, or if the CEL doesn't match the CDF
+    @since 2.4
+  */
+  public static void validateCelAndCdf(RawBioAssay rawBioAssay, ArrayDesign design, boolean required)
+  {
+    File celFile = getCelFile(rawBioAssay);
+    File cdfFile = getCdfFile(design);
+    if (celFile != null && cdfFile != null)
+    {
+      String cdfChipType = cdfFile.getName();
+      FusionCELData cel = loadCelFile(celFile);
+      FusionCDFData cdf = loadCdfFile(cdfFile);
+      validateCelAndCdf(cel, cdf, cdfChipType);
+    }
+    else if (required)
+    {
+      if (celFile == null)
+      {
+        throw new InvalidDataException("Raw bioassay has no CEL file: " + rawBioAssay);
+      }
+      else
+      {
+        throw new InvalidDataException("Array design has no CDF file: " + design);        
+      }
+    }
+  }
+  
+  private static void validateCelAndCdf(FusionCELData cel, FusionCDFData cdf, String cdfChipType)
+    throws InvalidDataException
+  {
+    String celChipType = cel.getChipType();
+    if (!cdfChipType.startsWith(celChipType))
+    {
+      throw new InvalidDataException("CEL chip type (" + celChipType + 
+        ") doesn't match CDF chip type (" + cdfChipType + ")");
+    }
+    
+    if (cel.getRows() != cdf.getHeader().getRows() || cel.getCols() != cdf.getHeader().getRows())
+    {
+      throw new InvalidDataException("CEL size (rows=" + cel.getRows() + ", cols=" + cel.getCols() +
+        ") doesn't match CDF size (rows=" + 
+        cdf.getHeader().getRows() + ", cols=" + cdf.getHeader().getCols()+")");
+    }   
   }
   

trunk/src/core/net/sf/basedb/core/Job.java

@@ -545 +545 @@
   {
     checkPermission(Permission.WRITE);
-    if (getStatus() != Status.WAITING) 
+    if (getStatus() != Status.WAITING && getStatus() != Status.UNCONFIGURED) 
     {
       throw new PermissionDeniedException("Can't prepare a job with status '"+getStatus()+"': "+toString());
@@ -577 +577 @@
     data.setStarted(new Date());
     data.setPercentComplete(0);
+  }
+  
+  /**
+    Get a progress reporter that reports progress by updating the information
+    in the database.
+    
+    @param chained An optional chained progress reporter to which all
+      updates will be forwarded
+    @return A progress reporter object
+    @since 2.4
+  */
+  public ProgressReporter getProgressReporter(ProgressReporter chained)
+  {
+    return new ProgressReporterImpl(this, null, chained);
   }
   

trunk/src/core/net/sf/basedb/core/RawBioAssay.java

@@ -24 +24 @@
 package net.sf.basedb.core;
 
+import net.sf.basedb.core.data.FeatureData;
 import net.sf.basedb.core.data.RawBioAssayData;
 import net.sf.basedb.core.data.RawData;
+import net.sf.basedb.core.data.ReporterData;
 import net.sf.basedb.core.data.SpotImagesData;
 
@@ -33 +35 @@
 import net.sf.basedb.core.query.Expressions;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.HashSet;
 import java.util.Collections;
 import java.util.TreeSet;
+
+import org.hibernate.EntityMode;
+import org.hibernate.metadata.ClassMetadata;
 
 /**
@@ -426 +433 @@
   
   /**
-    Set the {@link ArrayDesign} this raw data uses. The array design can't be changed
-    once raw data spots has been added.
-
+    Set the {@link ArrayDesign} this raw data uses. The array design can't be 
+    changed with this method once raw data spots has been added. Use
+    the {@link #updateArrayDesign(ArrayDesign, ProgressReporter)} to
+    do that.
+  
     @param arrayDesign The <code>ArrayDesign</code> item or null if not known
     @throws PermissionDeniedException If the logged in user doesn't have 
@@ -446 +455 @@
     getData().setArrayDesign(arrayDesign == null ? null : arrayDesign.getData());
   }
+  
+  /**
+    Set the {@link ArrayDesign} this raw data uses. If data has already been
+    imported to the raw bioassay, the reporters for each position on the 
+    new array design must match the positions of reporters already present in 
+    the raw bioassay.
+  
+    @param arrayDesign The <code>ArrayDesign</code> item or null if not known
+    @param progress An optional progress reporter
+    @throws PermissionDeniedException If the logged in user doesn't have 
+      write permission for the raw bioassay use permission for the array design
+    @throws InvalidDataException If the features of the new array design doesn't
+      match the raw data
+    @throws BaseException If there is another error
+    @since 2.4
+  */
+  public void updateArrayDesign(ArrayDesign arrayDesign, ProgressReporter progress)
+    throws PermissionDeniedException, InvalidDataException
+  {
+    checkPermission(Permission.WRITE);
+    if (arrayDesign != null) arrayDesign.checkPermission(Permission.USE);
+    if (getData().getSpots() > 0 && arrayDesign != null) 
+    {
+      // No need to validate if there is no change
+      if (arrayDesign.getData().equals(getData().getArrayDesign())) return;
+  
+      if (getRawDataType().isAffymetrix())
+      {
+        // Verify CEL and CDF files
+        Affymetrix.validateCelAndCdf(this, arrayDesign, false);
+      }
+      else if (getRawDataType().isStoredInDb())
+      {
+        validateFeatures(getDbControl(), arrayDesign, true, progress);
+      }
+    }
+    getData().setArrayDesign(arrayDesign == null ? null : arrayDesign.getData());
+  } 
 
   /**
@@ -687 +734 @@
   }
 
+  /**
+    Validate that the features/reporters of the raw data matches the 
+    features/reporter on the array design.
+    @param dc The DbControl to use for database access
+    @param design The array design to validate agains
+    @param update If the raw data should be linked to the new feature or not
+    @param progress An optional progress reporter
+    @since 2.4
+  */
+  private void validateFeatures(DbControl dc, ArrayDesign design, boolean update, ProgressReporter progress)
+  {
+    if (design.isAffyChip())
+    {
+      throw new InvalidDataException("The array design is an Affymetrix chip: " + design.getName());
+    }
+    if (design.hasFeatures())
+    {
+      if (progress != null) progress.display(5, "Loading features from array design...");
+      
+      // Hibernate metadata for setting new features
+      ClassMetadata metaData = HibernateUtil.getClassMetadata(getRawDataType().getEntityName());
+      org.hibernate.Session session = dc.getHibernateSession();
+      String entityName = getRawDataType().getEntityName();
+      
+      // Preload features from array design
+      org.hibernate.Query query = HibernateUtil.getPredefinedQuery(dc.getStatelessSession(), 
+        "COUNT_FEATURES_FOR_ARRAYDESIGN");
+      /*
+        SELECT count(*)
+        FROM FeatureData f
+        WHERE f.arrayDesignBlock.arrayDesign = :arrayDesign
+      */
+      query.setInteger("arrayDesign", design.getId());
+      int numFeatures = HibernateUtil.loadData(Long.class, query).intValue();
+      Map<Integer, FeatureData> features = new HashMap<Integer, FeatureData>(numFeatures);
+      query = HibernateUtil.getPredefinedQuery(dc.getStatelessSession(), "PRELOAD_FEATURES");
+      /*
+        SELECT f 
+        FROM FeatureData f 
+        JOIN FETCH f.arrayDesignBlock b 
+        LEFT JOIN FETCH f.reporter 
+        WHERE b.arrayDesign = :arrayDesign
+      */
+      query.setInteger("arrayDesign", design.getId());
+      ScrollIterator<FeatureData> si = HibernateUtil.loadIterator(FeatureData.class, query);
+      while (si.hasNext())
+      {
+        FeatureData f = si.next();
+        features.put(f.getPosition(), f);
+      }
+      si.close();
+      
+      // Load raw data
+      if (progress != null) progress.display(10, "Loading raw data from raw bioassay...");
+      DataQuery<RawData> rawQuery = getRawData();
+      DataResultIterator<RawData> result = rawQuery.iterate(dc);
+      int numValidated = 0;
+      int numTotal = getSpots();
+      while (result.hasNext())
+      {
+        RawData raw = result.next();
+        int position = raw.getPosition();
+        FeatureData feature = features.get(position);
+        if (feature == null)
+        {
+          throw new InvalidDataException("Array design '" + design.getName() +
+            "' is missing a feature at position " + raw.getPosition());
+        }
+        
+        // Check that reporters are the same
+        ReporterData rawReporter = raw.getReporter();
+        ReporterData featureReporter = feature.getReporter();
+        if (rawReporter == null)
+        {
+          if (featureReporter != null) 
+          {
+            throw new InvalidDataException("Reporter mismatch at position " + position +
+              "; raw reporter=none, feature reporter=" + featureReporter.getExternalId());
+          }
+        }
+        else
+        {
+          if (featureReporter == null)
+          {
+            throw new InvalidDataException("Reporter mismatch at position " + position +
+              "; raw reporter=" + rawReporter.getExternalId() + ", feature reporter=none");
+          }
+          else if (rawReporter.getId() != featureReporter.getId())
+          {
+            throw new InvalidDataException("Reporter mismatch at position " + position +
+              "; raw reporter=" + rawReporter.getExternalId() + 
+              ", feature reporter=" + featureReporter.getExternalId());
+          }
+        }
+          
+        // Check that coordinates are the same
+        FeatureCoordinate featureCoordinate = Feature.getFeatureCoordinate(feature);
+        FeatureCoordinate rawCoordinate = new FeatureCoordinate(raw.getBlock(), 
+          raw.getMetaGridX(), raw.getMetaGridY(), raw.getRow(), raw.getColumn());
+        
+        if (!featureCoordinate.equals(rawCoordinate))
+        {
+          throw new InvalidDataException("Coordinate mismatch at position " + position +
+            "; raw=" + rawCoordinate +", feature=" + featureCoordinate);
+        }
+        
+        // All is ok, re-assign raw data to the new feature
+        if (update)
+        {
+          metaData.setPropertyValue(raw, "feature", feature, EntityMode.POJO);
+          session.update(entityName, raw);
+        }
+        
+        // Report progress
+        numValidated++;
+        if (progress != null && numValidated % 100 == 0)
+        {
+          int percent = 10 + (90 * numValidated) / numTotal;
+          progress.display(percent, "Validated " + numValidated + " of " + numTotal + " spots.");
+        }
+      }
+    }
+  }
 }

trunk/src/core/net/sf/basedb/core/templates/hibernate-properties-RawData.xml

@@ -70 +70 @@
       cascade="evict"
       fetch="select"
-      update="false"
+      update="true"
       insert="true"
       column="`feature_id`"

trunk/www/views/jobs/view_job.jsp

@@ -421 +421 @@
       %>
       <%
-      if (job.getStatus() == Job.Status.ERROR)
+      if (job.getStatus() == Job.Status.ERROR && job.getJobType() == Job.Type.RUN_PLUGIN)
       {
         %>

trunk/www/views/rawbioassays/edit_rawbioassay.jsp

@@ -576 +576 @@
         <td>
           <%
-          if (rawBioAssay == null || !rawBioAssay.hasData())
+          //if (rawBioAssay == null || !rawBioAssay.hasData())
           {
             %>
@@ -593 +593 @@
             <%
           }
+          /*
           else
           {
@@ -601 +602 @@
             <%
           }
+          */
           %>
         </td>

trunk/www/views/rawbioassays/index.jsp

@@ -26 +26 @@
 --%>
 <%@ page session="false"
+  import="net.sf.basedb.core.Application"
   import="net.sf.basedb.core.SessionControl"
   import="net.sf.basedb.core.DbControl"
@@ -51 +52 @@
   import="net.sf.basedb.core.PluginDefinition"
   import="net.sf.basedb.core.Job"
+  import="net.sf.basedb.core.ProgressReporter"
   import="net.sf.basedb.core.IntegerParameterType"
   import="net.sf.basedb.core.FloatParameterType"
@@ -69 +71 @@
   import="java.util.List"
   import="java.util.ArrayList"
+  import="java.util.Arrays"
   import="java.util.Collections"
 %>
@@ -75 +78 @@
   private static final ItemContext defaultContext = Base.createDefaultContext("name", "name,rawDataType,spots,description");
   private static final Item itemType = Item.RAWBIOASSAY;
+  
+  private static class UpdateArrayDesign
+    implements Runnable
+  {
+    private final int jobId;
+    private final int rawBioAssayId;
+    private final int arrayDesignId;
+    private final SessionControl sc;
+    
+    private UpdateArrayDesign(Job job, RawBioAssay rba, ArrayDesign design)
+    {
+      this.jobId = job.getId();
+      this.rawBioAssayId = rba.getId();
+      this.arrayDesignId = design.getId();
+      this.sc = job.getSessionControl();
+    }
+    
+    public void run()
+    {
+      DbControl dc = sc.newDbControl();
+      try
+      {
+        Job j = Job.getById(dc, jobId);
+        j.start("Initialising...", Application.getHostName());
+        dc.commit();
+        dc = sc.newDbControl();
+        RawBioAssay rba = RawBioAssay.getById(dc, rawBioAssayId);
+        ArrayDesign design = ArrayDesign.getById(dc, arrayDesignId);
+        ProgressReporter progress = j.getProgressReporter(null);
+        try
+        {
+          rba.updateArrayDesign(design, progress);
+          progress.display(99, "Committing changes to database...");
+          dc.commit();
+          dc = sc.newDbControl();
+          j = Job.getById(dc, jobId);
+          j.doneOk("Array design updated successfully");
+          j.setRemoved(true);
+          dc.commit();
+        }
+        catch (Throwable t)
+        {
+          dc = sc.newDbControl();
+          j = Job.getById(dc, jobId);
+          j.doneError(t.getMessage(), Arrays.asList(t));
+          j.setRemoved(true);
+          dc.commit();
+        }
+      }
+      finally
+      {
+        if (dc != null) dc.close();
+      }
+    }
+  }
 %>
 <%
@@ -192 +250 @@
       if (scan != null) cc.setRecent(scan, maxRecent);
     }
+    Job job = null;
+    ArrayDesign ad = null;
     int arrayDesignId = Values.getInt(request.getParameter("arraydesign_id"), -1);
     if (arrayDesignId >= 0) // < 0 = denied or unchanged
     {
-      ArrayDesign ad = arrayDesignId == 0 ? null : ArrayDesign.getById(dc, arrayDesignId);
-      rba.setArrayDesign(ad);
+      ad = arrayDesignId == 0 ? null : ArrayDesign.getById(dc, arrayDesignId);
+      if (rba.getSpots() > 0)
+      {
+        if (rba.getRawDataType().isStoredInDb() && ad != null && ad.hasFeatures())
+        {
+          // Create job and assign it to another thread
+          job = Job.getNew(dc, null, null);
+          job.setName("Validating array design on " + rba.getName());
+          job.setPrepared(Application.getHostName());
+          dc.saveItem(job);
+        }
+        else
+        {
+          rba.updateArrayDesign(ad, null);
+        }
+      }
+      else
+      {
+        rba.setArrayDesign(ad);
+      }
       if (ad != null) cc.setRecent(ad, maxRecent);
     }
@@ -214 +292 @@
     dc.commit();
     cc.removeObject("item");
+    if (job != null) 
+    {
+      redirect = "../jobs/index.jsp?ID="+ID+"&cmd=ViewItem&item_id="+job.getId();
+      Thread updateThread = new Thread(new UpdateArrayDesign(job, rba, ad));
+      updateThread.setPriority(Thread.currentThread().getPriority() - 1);
+      updateThread.start();
+    }
   }
   else if ("DeleteItem".equals(cmd))