Changeset 5843

Timestamp:

Feb 25, 2020, 2:21:49 PM (3 years ago)

Author:

Nicklas Nordborg

Message:

References #1218: Implement MIPs alignment

Importing some numbers from the metrics files to annotations. Existing annotations:

• ALIGNED_PAIRS
• READ_PAIRS_EXAMINED
• READ_PAIR_DUPLICATES
• FRACTION_DUPLICATION

New annotation:

• ALIGNED_BASES

Location:

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie

Files:

• Reggie.java (modified) (1 diff)
• dao/Annotationtype.java (modified) (1 diff)
• grid/MipsAlignJobCreator.java (modified) (6 diffs)
• servlet/InstallServlet.java (modified) (2 diffs)

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/Reggie.java

@@ -300 +300 @@
   }
     
-  public static String formatCount(long count)
-  {
-    if (count > 1000000)
-    {
-      int numDecimals = count < 100000000 ? 1 : 0;
-      return Values.formatNumber(count / 1000000f, numDecimals, "M");
-    }
-    else if (count > 1000)
-    {
-      int numDecimals = count < 100000 ? 1 : 0;
-      return Values.formatNumber(count / 1000f, numDecimals, "k");
-    }
-    return Long.toString(count);
+  public static String formatCount(Number count)
+  {
+    if (count == null) return "";
+    long c = count.longValue();
+    if (c > 1000000)
+    {
+      int numDecimals = c < 100000000 ? 1 : 0;
+      return Values.formatNumber(c / 1000000f, numDecimals, "M");
+    }
+    else if (c > 1000)
+    {
+      int numDecimals = c < 100000 ? 1 : 0;
+      return Values.formatNumber(c / 1000f, numDecimals, "k");
+    }
+    return Long.toString(c);
   }
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Annotationtype.java

@@ -1552 +1552 @@
   public static final Annotationtype ALIGNED_PAIRS =
     new Annotationtype("ALIGNED_PAIRS", Type.LONG, false, Item.DERIVEDBIOASSAY);
+
+  /**
+    The "ALIGNED_BASES" annotation, used for derived bioassays (AlignedSequences).
+    The annotation is the number of bases that has been aligned.
+    @since 4.26
+  */
+  public static final Annotationtype ALIGNED_BASES =
+    new Annotationtype("ALIGNED_BASES", Type.LONG, false, Item.DERIVEDBIOASSAY);
+
   
   /**

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/MipsAlignJobCreator.java

@@ -2 +2 @@
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
@@ -409 +410 @@
       script.cmd("rm -rf ${AlignFolder}/*");
       script.cmd("cp out/* ${AlignFolder}");
-      // TODO - copy some files to $WD so that we can update some annotations like ALIGNED_PAIRS, etc.
+      script.cmd("cp out/concordant.*_metrics.txt ${WD}");
       if (externalGroup != null)
       {
@@ -446 +447 @@
       String jobName = status.getName();
       String files = session.getJobFileAsString(jobName, "files.out", "UTF-8");
-
-      Metrics metrics = parseAlignedOut(sc, job, files);
-      return null;
+      String alignmentMetrics = session.getJobFileAsString(jobName, "concordant.alignment_summary_metrics.txt", "UTF-8");
+      String duplicateMetrics = session.getJobFileAsString(jobName, "concordant.dedup_metrics.txt", "UTF-8");
+
+      Metrics metrics = parseAlignedOut(sc, job, files, alignmentMetrics, duplicateMetrics);
+      String msg = Reggie.formatCount(metrics.alignedReadPairs) + " reads after alignment; ";
+      msg += Values.formatNumber(metrics.fractionDuplication * 100,  1) + "% duplicates; ";
+      msg += Reggie.formatCount(metrics.alignedBases) + " aligned bases";
+      return msg;
     }
     
-    private Metrics parseAlignedOut(SessionControl sc, Job job, String filesOut)
+    private Metrics parseAlignedOut(SessionControl sc, Job job, String filesOut, String alignmentMetrics, String duplicateMetrics)
     {
       Metrics metrics = new Metrics();
+
+      int categoryIndex = -1;
+      int alignedPairsIndex = -1;
+      int alignedBasesIndex = -1;
+      for (String line : alignmentMetrics.split("\n"))
+      {
+        String[] cols = line.split("\t");
+        if (cols.length >= 20)
+        {
+          if (categoryIndex == -1)
+          {
+            List<String> colsA = Arrays.asList(cols);
+            categoryIndex = colsA.indexOf("CATEGORY");
+            alignedPairsIndex = colsA.indexOf("READS_ALIGNED_IN_PAIRS");
+            alignedBasesIndex = colsA.indexOf("PF_ALIGNED_BASES");
+          }
+          else if ("PAIR".equals(cols[categoryIndex]))
+          {
+            Long tmp = Values.getLong(cols[alignedPairsIndex], null);
+            if (tmp != null) metrics.alignedReadPairs = tmp / 2;
+            metrics.alignedBases = Values.getLong(cols[alignedBasesIndex], null);
+            break;
+          }
+        }
+      }
+
+      
+      int readPairsExaminedIndex = -1;
+      int readPairDuplicatesIndex = -1;
+      int percentDuplicationIndex = -1;
+      for (String line : duplicateMetrics.split("\n"))
+      {
+        String[] cols = line.split("\t");
+        if (cols.length >= 9)
+        {
+          if (readPairsExaminedIndex == -1)
+          {
+            List<String> colsA = Arrays.asList(cols);
+            readPairsExaminedIndex = colsA.indexOf("READ_PAIRS_EXAMINED");
+            readPairDuplicatesIndex = colsA.indexOf("READ_PAIR_DUPLICATES");
+            percentDuplicationIndex = colsA.indexOf("PERCENT_DUPLICATION");
+          }
+          else
+          {
+            metrics.readPairsExamined = Values.getLong(cols[readPairsExaminedIndex], null);
+            metrics.readPairDuplicates = Values.getLong(cols[readPairDuplicatesIndex], null);
+            metrics.fractionDuplication = Values.getFloat(cols[percentDuplicationIndex], null);
+            break;
+          }
+        }
+      }
       
       DbControl dc = null;
@@ -462 +519 @@
         AlignedSequences alignedSequences = AlignedSequences.getByJob(dc, job);
         DerivedBioAssay aligned = alignedSequences.getItem();
+
+        Annotationtype.ALIGNED_PAIRS.setAnnotationValue(dc, aligned, metrics.alignedReadPairs);
+        Annotationtype.ALIGNED_BASES.setAnnotationValue(dc, aligned, metrics.alignedBases);
+        Annotationtype.READ_PAIRS_EXAMINED.setAnnotationValue(dc, aligned, metrics.readPairsExamined);
+        Annotationtype.READ_PAIR_DUPLICATES.setAnnotationValue(dc, aligned, metrics.readPairDuplicates);
+        Annotationtype.FRACTION_DUPLICATION.setAnnotationValue(dc, aligned, metrics.fractionDuplication);
 
         // Create file links
@@ -497 +560 @@
           if (f.getName().equals("concordant.bam"))
           {
-            //f.setDescription(metrics.numReadsAfterAlign + " ALIGNED PAIRS");
+            f.setDescription(metrics.alignedReadPairs + " ALIGNED PAIRS; " + metrics.alignedBases + " ALIGNED BASES");
             f.setItemSubtype(bamType);
             FileSetMember member = aligned.getFileSet().addMember(f, bamData);
@@ -522 +585 @@
   static class Metrics
   {
+    Long alignedReadPairs = null;
+    Long alignedBases = null;
+    Long readPairsExamined = null;
+    Long readPairDuplicates = null;
+    Float fractionDuplication = null;
   }
 

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/InstallServlet.java

@@ -628 +628 @@
         jsonChecks.add(checkAnnotationType(dc, Annotationtype.PM_READS, 1, null, createIfMissing, effectivePermissionsUse));
         jsonChecks.add(checkAnnotationType(dc, Annotationtype.ALIGNED_PAIRS, 1, null, createIfMissing, effectivePermissionsUse));
+        jsonChecks.add(checkAnnotationType(dc, Annotationtype.ALIGNED_BASES, 1, null, createIfMissing, effectivePermissionsUse));
         jsonChecks.add(checkAnnotationType(dc, Annotationtype.READ_PAIRS_EXAMINED, 1, null, createIfMissing, effectivePermissionsUse));
         jsonChecks.add(checkAnnotationType(dc, Annotationtype.READ_PAIR_DUPLICATES, 1, null, createIfMissing, effectivePermissionsUse));
@@ -965 +966 @@
         jsonChecks.add(checkAnnotationTypeCategory(dc, Subtype.ALIGNED_SEQUENCES, createIfMissing, 
             Annotationtype.DATA_FILES_FOLDER, Annotationtype.ALIGNED_PAIRS,
+            Annotationtype.ALIGNED_BASES,
             Annotationtype.READ_PAIRS_EXAMINED, Annotationtype.READ_PAIR_DUPLICATES,
             Annotationtype.FRACTION_DUPLICATION, Annotationtype.FRAGMENT_SIZE_AVG,