Changeset 3485

Timestamp:

Sep 15, 2015, 8:13:19 AM (8 years ago)

Author:

Nicklas Nordborg

Message:

References #809: Improve performance of demux+merge step

Added time measurements to the generated job script. It will output the current time after most steps and save that to the 'time.txt' file in the job results folder. The file is not imported back to BASE.

Location:

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie

Files:

• servlet/DemuxMergeServlet.java (modified) (9 diffs)
• ssh/ScriptBuilder.java (modified) (1 diff)

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java

@@ -665 +665 @@
             // Start scripting for demux
             script.comment("Demuxing " + demuxName);
+            script.time("STARTING: Demux " + demuxName);
             // Actual 'dataFolder' may be in 'run_archive/current' or 'run_archive/YYYY', so we
             // need to find the actual folder first
@@ -724 +725 @@
               extractCmd += " >> " + demuxName + "_" + lane + ".out";
               script.cmd(extractCmd);
+              script.time("DONE: ExtractIlluminaBarcodes; lane="+lane);
               
               // Copy demultiplex metrics back to job folder
@@ -759 +761 @@
               fastqCmd += " >> " + demuxName + "_" + lane + ".out";
                 script.cmd(fastqCmd);
+              script.time("DONE: IlluminaBasecallsToFastq; lane="+lane);
                 if (earlyCleanup)
                 {
@@ -774 +777 @@
             script.cmd("rm -f fastq/AD0*");
           }
+          
+          script.time("DEMUX COMPLETE");
           
           // Generate script for moving FASTQ files to project-archive
@@ -795 +800 @@
             
             script.progress(percent, "Consolidating FASTQ: " + mergeName);
+            script.time("STARTING: Merge " + mergeName);
             script.cmd("echo [" + mergeName + "] >> " + trimmomaticOut);
             script.cmd("echo [" + mergeName + "] >> " + filesOut);
@@ -805 +811 @@
               script.cmd("rm -f fastq/" + mergeName + "*");
             }
+            script.time("DONE: Merge " + mergeName);
             script.newLine();
 
@@ -820 +827 @@
             script.cmd(alignCmd);
             script.cmd("./singlecolumnaverager.awk < fastq.aligned/" + SAM_name + " > fastq.aligned/" + fragments_name);
+            script.time("DONE: Bowtie2 " + mergeName);
             script.newLine();
             
@@ -853 +861 @@
             trimCmd2 += " >> "+trimmomaticOut;
             script.cmd(trimCmd1);
+            script.time("DONE: Trimmomatic 1 " + mergeName);
+
             script.cmd(trimCmd2);
+            script.time("DONE: Trimmomatic 2 " + mergeName);
             script.newLine();
             
@@ -862 +873 @@
             script.cmd("cp fastq.aligned/"+fragments_name + " " + fastqFolder + "/" + fragments_name);
             script.cmd("gzip -c fastq.trimmomatic.2/" + R1_name + " > " + fastqFolder + "/"+R1_name + ".gz");
+            script.time("DONE: gzip " + R1_name);
             script.cmd("gzip -c fastq.trimmomatic.2/" + R2_name + " > " + fastqFolder + "/"+R2_name + ".gz");
+            script.time("DONE: gzip " + R2_name);
 
             script.cmd("ls -1 "+fastqFolder+"/*.fastq.gz >> " + filesOut);

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/ssh/ScriptBuilder.java

@@ -54 +54 @@
   {
     script.append("\n");
+  }
+  
+  /**
+    Output the comment and current time (hour:minute:second) into the 'time.txt' file in the
+    job folder. Eg: 'Demux complete: 12:23:45'.
+    @since 3.6
+  */
+  public void time(String comment)
+  {
+    script.append("echo \"").append(comment).append(": `date +%H:%M:%S`\" >> ").append(jobFolder).append("/time.txt\n");
   }