Changeset 4246

Timestamp:

Apr 24, 2008, 11:39:07 AM (15 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #968: Raw data importer – Enhance feature mismatch error handling

Location:

trunk

Files:

• doc/test/roles/index.html (modified) (6 diffs)
• src/core/net/sf/basedb/core/RawDataBatcher.java (modified) (5 diffs)
• src/plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java (modified) (6 diffs)
• src/test/TestArrayDesign.java (modified) (3 diffs)
• src/test/TestDirty.java (modified) (1 diff)
• src/test/TestExperiment.java (modified) (1 diff)
• src/test/TestRawBioAssay.java (modified) (1 diff)
• src/test/TestRawDataFlatFileImporter.java (modified) (2 diffs)
• src/test/TestWebservices.java (modified) (1 diff)
• src/test/net/sf/basedb/test/roles/AnalysisTest.java (modified) (1 diff)
• src/test/net/sf/basedb/test/roles/UserTest.java (modified) (1 diff)

trunk/doc/test/roles/index.html

@@ -1606 +1606 @@
       <th>Raw bioassay</th>
       <th>Raw data file</th>
-      <th>Spots inserted</th>
+      <th>Spots inserted/with null reporter/skipped)</th>
       <th>Annotations created</th>
     </tr>
@@ -1612 +1612 @@
       <td>Raw bioassay A.00h</td>
       <td>genepix.mouse.v4.37k.00h.gpr</td>
-      <td>36,232</td>
+      <td>36,864/632/768</td>
       <td><i>Dye swap</i>: false</td>
     </tr>
@@ -1618 +1618 @@
       <td>Raw bioassay A.24h</td>
       <td>genepix.mouse.v4.37k.24h.gpr</td>
-      <td>36,232</td>
+      <td>36,864/632/768</td>
       <td><i>Dye swap</i>: false</td>
     </tr>   
@@ -1624 +1624 @@
       <td>Raw bioassay A.00h (dye-swap)</td>
       <td>genepix.mouse.v4.37k.00h.dyeswap.gpr</td>
-      <td>36,232</td>
+      <td>36,864/632/768</td>
       <td><i>Dye swap</i>: true</td>
     </tr> 
@@ -1630 +1630 @@
       <td>Raw bioassay A.24h (dye-swap)</td>
       <td>genepix.mouse.v4.37k.24h.dyeswap.gpr</td>
-      <td>36,232</td>
+      <td>36,864/632/768</td>
       <td><i>Dye swap</i>: true</td>
     </tr>
@@ -1751 +1751 @@
       <td>Filtered bioassay set</td>
       <td>All intensities &gt; 0</td>
-      <td>ch(1) &gt; 0 &amp;&amp; ch(2) &gt; 0</td>
+      <td>ch(1) &gt; 0 &amp;&amp; ch(2) &gt; 0 &amp;&amp; rep('id') != null</td>
     </tr>
     </table>
     Wait for the plug-in to finish. It should report that 136,498 spots remain and 
-    that 8,430 spots has been removed.
+    that 10,958 spots has been removed.
     <p>
   </li>

trunk/src/core/net/sf/basedb/core/RawDataBatcher.java

@@ -176 +176 @@
   private boolean useNullIfReporterNotFound = true;
   
+  /**
+    The number of reporters that was not found.
+  */
+  private int numReportersNotFound = 0;
+  
+  /**
+    If inserts that can't find a matching feature should be skipped
+    or generate an error.
+  */
+  private boolean skipInsertIfMissingFeature = false;
+
+  /**
+    The number of skipped inserts due to missing feature
+  */ 
+  private int numSkippedMissingFeature = 0;
+  
+  /**
+    If we should always accept inserts when the feature has a NULL
+    reporter.
+  */
+  private boolean acceptInsertIfNullReporterOnFeature = false;
+  
+  /**
+    The number of accepted inserts were the feature had a NULL reporter
+  */ 
+  private int numAcceptedWithNullReporterOnFeature = 0;
+
   RawDataBatcher(DbControl dc, RawBioAssay rawBioAssay, FeatureIdentificationMethod fim)
     throws BaseException
@@ -376 +403 @@
     Specify if the insert method should use a null reporter if a reporter
     with the given external ID isn't found. This setting only affects raw bioassays that
-    are not connected to an array design. The insert always fails if the reporter
-    doesn't match the array design.
+    are not connected to an array design.
     @param useNull TRUE if the insert should use a null reporter (default), 
       FALSE otherwise
+    @see #skipInsertIfMissingFeature(boolean)
   */
   public void useNullIfReporterNotFound(boolean useNull)
@@ -385 +412 @@
     this.useNullIfReporterNotFound = useNull;
   }
+  
+  /**
+    Get the number of reporters that was not found.
+    @since 2.7
+    @see #useNullIfReporterNotFound(boolean)
+  */
+  public int getNumReportersNotFound()
+  {
+    return numReportersNotFound;
+  }
+  
+  /**
+    Specify if the insert method should skip the insert if a feature for
+    the spot can't be found. This setting only affects raw bioassays that
+    are connected to an array design.
+    
+    @param skip TRUE if the insert should be skipped, FALSE to
+      throw an {@link ItemNotFoundException} (default)
+    @since 2.7
+    @see #useNullIfReporterNotFound(boolean)
+  */
+  public void skipInsertIfMissingFeature(boolean skip)
+  {
+    this.skipInsertIfMissingFeature = skip;
+  }
+
+  /**
+    Get the number of skipped inserts because of a missing feature.
+    @since 2.7
+    @see #skipInsertIfMissingFeature(boolean)
+  */
+  public int getNumSkippedMissingFeature()
+  {
+    return numSkippedMissingFeature;
+  }
+  
+  /**
+    Specify if inserts always should be accepted if there is a matching 
+    feature that has a NULL reporter. This settings only affects raw 
+    bioassays that are connected to an array design.
+    <p>
+    NOTE! The inserted data will also reference the NULL reporter. 
+    
+    @param accept TRUE to always accept the insert, FALSE to throw an 
+      {@link ItemNotFoundException} if the insert doesn't specify a NULL
+      reporter (default)
+    @since 2.7
+  */
+  public void acceptInsertIfNullReporterOnFeature(boolean accept)
+  {
+    this.acceptInsertIfNullReporterOnFeature = accept;
+  }
+  
+  /**
+    Get the number of accepted inserts because of features with
+    a NULL reporter.
+    @since 2.7
+    @see #acceptInsertIfNullReporterOnFeature(boolean)
+  */
+  public int getNumAcceptedWithNullReporterOnFeature()
+  {
+    return numAcceptedWithNullReporterOnFeature;
+  } 
   
   /**
@@ -462 +552 @@
         findReporter.setString("externalId", externalReporterId);
         ReporterData reporter = HibernateUtil.loadData(ReporterData.class, findReporter);
-        if (reporter == null && !useNullIfReporterNotFound)
+        if (reporter == null)
         {
-          throw new ItemNotFoundException("Reporter[externalId=" + externalReporterId + "]");
+          if (useNullIfReporterNotFound)
+          {
+            numReportersNotFound++;
+          }
+          else
+          {
+            throw new ItemNotFoundException("Reporter[externalId=" + externalReporterId + "]");
+          }
         }
         data.setReporter(reporter);
@@ -477 +574 @@
           "] has already been used by another spot.");
       }
-      FeatureData f = preloaded.get(featureId);
-      if (f == null)
-      {
-        throw new ItemNotFoundException("Feature["+ fiMethod.toString() + "=" + featureId + "] doesn't exist on array design");
-      }
-      ReporterData r = f.getReporter();
-      if (validateReporterId)
+      FeatureData feature = preloaded.get(featureId);
+      if (feature == null)
+      {
+        numSkippedMissingFeature++;
+        if (skipInsertIfMissingFeature) return;
+        throw new ItemNotFoundException("Feature["+ fiMethod.toString() + 
+            "=" + featureId + "] doesn't exist on array design");
+      }
+      ReporterData reporterOnFeature = feature.getReporter();
+      if (reporterOnFeature == null && acceptInsertIfNullReporterOnFeature)
+      {
+        numAcceptedWithNullReporterOnFeature++;
+      }
+      else if (validateReporterId)
       {
         // Get the reporter from the feature and verify that it has the same externalId
-        String rId = r == null ? null : r.getExternalId();
-        if (!StringUtil.isEqualOrNull(externalReporterId, rId, caseInsensitive))
+        String reporterIdOnFeature = reporterOnFeature == null ? null : reporterOnFeature.getExternalId();
+        if (!StringUtil.isEqualOrNull(externalReporterId, reporterIdOnFeature, caseInsensitive))
         {
           throw new ItemNotFoundException("Reporter mismatch: The feature has reporter '" +
-            rId + "' whereas you have given '" + externalReporterId + "'");
+            reporterIdOnFeature + "' whereas you have given '" + externalReporterId + "'");
         }
       }
-      data.setPosition(f.getPosition());
-      setPropertyValue(data, "feature", f);
-      data.setReporter(r);
+      data.setPosition(feature.getPosition());
+      setPropertyValue(data, "feature", feature);
+      data.setReporter(reporterOnFeature);
     }
     super.insert(data);

trunk/src/plugins/core/net/sf/basedb/plugins/RawDataFlatFileImporter.java

@@ -237 +237 @@
       "This can for example be caused by invalid block and/or spot coordinates or a different reporter. " +
       "If not specified the default error handling is used.\n\n"+
+      "smart = SKIP lines that have no feature on the array design, " +
+      "ACCEPT lines that has a feature with a NULL reporter, " +
+      "FAIL lines that has a feature with different reporter\n" +
       "skip = Skip the current data line and continue\n"+
       "fail = Stop with an error message",
-      new StringParameterType(255, null, false, 1, 0, 0, 
-        Arrays.asList( new String[] { "skip", "fail"} ))
+      new StringParameterType(255, "smart", false, 1, 0, 0, 
+        Arrays.asList( new String[] { "smart", "skip", "fail"} ))
     );
 
@@ -272 +275 @@
 
   private DbControl dc;
-  private int numInserted;
   private FlatFileParser ffp;
   private NumberFormat numberFormat;
@@ -561 +563 @@
     
     batcher = rawBioAssay.getRawDataBatcher(fiMethod);
-    numInserted = 0;
     this.ffp = ffp;
     this.numberFormat = ffp.getDefaultNumberFormat();
@@ -573 +574 @@
       {
         addErrorHandler(ItemNotFoundException.class, new SimpleErrorHandler("skip".equals(method)));
+        boolean isSmart = "smart".equals(method);
+        batcher.skipInsertIfMissingFeature(isSmart);
+        batcher.acceptInsertIfNullReporterOnFeature(isSmart);
       }
     }
@@ -680 +684 @@
       raw.setExtended(ep.getName(), ep.parseString(m.getValue(data), numberFormat, nullIfException));
     }
-
     batcher.insert(raw, externalId, featureId);
-    numInserted++;
   }
   
@@ -700 +702 @@
   protected String getSuccessMessage(int skippedLines)
   {
-    String skipped = skippedLines > 0 ? "; " + skippedLines + " spot(s) skipped due to errors": "";
-    return numInserted + " spot(s) inserted" + skipped;
+    StringBuilder msg = new StringBuilder();
+    msg.append(batcher.getTotalInsertCount()).append(" spot(s) inserted");
+    int numReportersNotFound = batcher.getNumReportersNotFound();
+    int numSkippedMissingFeature = batcher.getNumSkippedMissingFeature();
+    int numAcceptedWithNullReporterOnFeature = batcher.getNumAcceptedWithNullReporterOnFeature();
+    if (numReportersNotFound > 0)
+    {
+      msg.append("; ").append(numReportersNotFound).append(" spot(s) could not find a reporter");
+    }
+    if (numAcceptedWithNullReporterOnFeature > 0)
+    {
+      msg.append("; ").append(numAcceptedWithNullReporterOnFeature);
+      msg.append(" spot(s) accepted with null reporter on feature");
+    }
+    if (numSkippedMissingFeature > 0)
+    {
+      msg.append("; ").append(numSkippedMissingFeature);
+      msg.append(" spot(s) skipped due to missing features");
+    }
+    if (skippedLines > 0) 
+    {
+      msg.append("; ").append(skippedLines).append(" spot(s) skipped due to errors");
+    }
+    return msg.toString();
   }
   // -------------------------------------------

trunk/src/test/TestArrayDesign.java

@@ -83 +83 @@
       "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"ID\"\\t\"Name\".*", "\\t", 3, 4);
     test_import_from_file(id2, "data/test.reportermap.import.txt", 
-      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"ID\"\\t\"Name\".*", 0, 2, 1, 3);
+      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"ID\"\\t\"Name\".*", null, 0, 2, 1, 3);
     test_add_features(id, FeatureIdentificationMethod.COORDINATES, 10);
     test_add_features(id6, FeatureIdentificationMethod.FEATURE_ID, 10);
@@ -547 +547 @@
   }
 
-  static void test_import_from_file(int arrayDesignId, String filename, String headerRegexp, int blockCol, int rowCol, int colCol, int reporterCol)
+  static void test_import_from_file(int arrayDesignId, String filename, String headerRegexp, 
+      String ignoreRegexp, int blockCol, int rowCol, int colCol, int reporterCol)
   {
     if (arrayDesignId == 0) return;
@@ -564 +565 @@
       parser.setDataHeaderRegexp(Pattern.compile(headerRegexp));
       parser.setDataSplitterRegexp(Pattern.compile("\\t"));
+      if (ignoreRegexp != null)
+      {
+        parser.setIgnoreRegexp(Pattern.compile(ignoreRegexp));
+      }
       parser.setInputStream(FileUtil.getInputStream(new java.io.File(filename)), "ISO-8859-1");
       parser.parseHeaders();

trunk/src/test/TestDirty.java

@@ -75 +75 @@
     int arrayDesignId = TestArrayDesign.test_create(Platform.GENERIC, false);
     TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.import.dirty.txt", 
-        "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);
+        "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", null, 0, 2, 1, 4);
     
     int rawBioAssayId = TestRawBioAssay.test_create(Platform.GENERIC, "genepix", 0, 0, 0, arrayDesignId, false);

trunk/src/test/TestExperiment.java

@@ -85 +85 @@
     int arrayDesignId = TestArrayDesign.test_create(Platform.GENERIC, false);
     TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.rawdata.import.txt", 
-      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);
+      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", null, 0, 2, 1, 4);
     
     int rbaId1 = TestRawBioAssay.test_create(Platform.GENERIC, "genepix", 0, 0, 0, arrayDesignId, false);

trunk/src/test/TestRawBioAssay.java

@@ -58 +58 @@
       "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", "\\t", 4, 3);
     TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.rawdata.import.txt", 
-      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);
+      "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", null, 0, 2, 1, 4);
     int softwareId = TestSoftware.test_create(SystemItems.getId(SoftwareType.FEATURE_EXTRACTION), false);
     int protocolId = TestProtocol.test_create(SystemItems.getId(ProtocolType.FEATURE_EXTRACTION), false);

trunk/src/test/TestRawDataFlatFileImporter.java

@@ -46 +46 @@
     int fileId = TestFile.test_create("data/test.rawdata.import.txt", false, false);
     int arrayDesignId = TestArrayDesign.test_create(Platform.GENERIC, false);
-    TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.rawdata.import.txt", "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);
+    TestArrayDesign.test_import_from_file(arrayDesignId, "data/test.rawdata.import.txt", 
+        "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", ".*(EMPTY_WELL|no clone).*", 0, 2, 1, 4);
     int rawBioAssayId = TestRawBioAssay.test_create(Platform.GENERIC, "genepix", 0, 0, 0, arrayDesignId, true);
     
@@ -191 +192 @@
       request.setParameterValue("decimalSeparator", "dot");
       request.setParameterValue("duplicateFeatureError", "skip");
+      request.setParameterValue("featureMismatchError", "smart");
 
       PluginResponse response = request.invoke();

trunk/src/test/TestWebservices.java

@@ -104 +104 @@
     int arrayDesignId1 = TestArrayDesign.test_create(Platform.GENERIC, false);
     TestArrayDesign.test_import_from_file(arrayDesignId1, "data/test.rawdata.import.txt", 
-        "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", 0, 2, 1, 4);    
+        "\"Block\"\\t\"Column\"\\t\"Row\"\\t\"Name\"\\t\"ID\".*", null, 0, 2, 1, 4);    
     int rawBioAssayId1 = TestRawBioAssay.test_create(Platform.GENERIC, "genepix", 0, 0, 0, arrayDesignId1, false);
     int rawBioAssayId2 = TestRawBioAssay.test_create(Platform.GENERIC, "genepix", 0, 0, 0, arrayDesignId1, false);

trunk/src/test/net/sf/basedb/test/roles/AnalysisTest.java

@@ -97 +97 @@
         "Root bioassay set (" + user + ")", formula);
       BioAssaySet filtered = filterBioAssaySet(experiment, root, 
-        "Filtered bioassay set (" + user + ")", "ch(1) > 0 && ch(2) > 0");
+        "Filtered bioassay set (" + user + ")", "ch(1) > 0 && ch(2) > 0 && rep('id') != null");
 
       BioAssaySet normalized = normalizeBioAssaySet(experiment, filtered, 

trunk/src/test/net/sf/basedb/test/roles/UserTest.java

@@ -372 +372 @@
       job.setParameterValue("rawBioAssay", 
         new ItemParameterType<RawBioAssay>(RawBioAssay.class, null), rba);
-      job.setParameterValue("missingReporterError", new StringParameterType(), "skip");
-      job.setParameterValue("featureMismatchError", new StringParameterType(), "skip");
+      job.setParameterValue("missingReporterError", new StringParameterType(), "null");
+      job.setParameterValue("featureMismatchError", new StringParameterType(), "smart");
       dc.saveItem(job);
       dc.commit();