Changeset 3487

Timestamp:

Sep 16, 2015, 11:08:24 AM (8 years ago)

Author:

Nicklas Nordborg

Message:

References #809: Improve performance of demux+merge step

Switched to pigz instead of gzip. Options can be set in reggie-config.xml inside <demux><pigz-options> tag. By default this only set the compression level to -5.

Location:

extensions/net.sf.basedb.reggie/trunk

Files:

• config/reggie-config.xml (modified) (1 diff)
• src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java (modified) (2 diffs)

extensions/net.sf.basedb.reggie/trunk/config/reggie-config.xml

@@ -172 +172 @@
           <step-2>MAXINFO:40:0.9 MINLEN:20</step-2>
         </trimmomatic-options>
+        <!-- static options for gzip compression with pigz (default=-5) -->
+        <!-- NOTE! Number of threads (-p) is set automatically and should not be included here -->
+        <pigz-options>-5</pigz-options>
       </demux>
   

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java

@@ -425 +425 @@
         String demux_trimmomaticOptions1 = cfg.getRequiredConfig("demux/trimmomatic-options/step-1", mergeParameterSet);
         String demux_trimmomaticOptions2 = cfg.getRequiredConfig("demux/trimmomatic-options/step-2", mergeParameterSet);
+        String pigzOptions = cfg.getConfig("demux/pigz-options", mergeParameterSet, "-5");
         
         List<DemuxBatch> demuxBatch = new ArrayList<DemuxBatch>();
@@ -872 +873 @@
             script.cmd("cat fastq.aligned/" + fragments_name + " >> " + fragmentsOut);
             script.cmd("cp fastq.aligned/"+fragments_name + " " + fastqFolder + "/" + fragments_name);
-            String pid1 = script.bkgr("gzip -1 -c fastq.trimmomatic.2/" + R1_name + " > " + fastqFolder + "/"+R1_name + ".gz");
-            String pid2 = script.bkgr("gzip -1 -c fastq.trimmomatic.2/" + R2_name + " > " + fastqFolder + "/"+R2_name + ".gz");
-            script.waitForProcess(pid1);
-            script.waitForProcess(pid2);
-            script.time("DONE: gzip " + mergeName);
+            script.cmd("pigz " + pigzOptions + " -p ${NumThreads} -c fastq.trimmomatic.2/" + R1_name + " > " + fastqFolder + "/"+R1_name + ".gz");
+            script.time("DONE: pigz " + R1_name);
+            script.cmd("pigz " + pigzOptions + " -p ${NumThreads} -c fastq.trimmomatic.2/" + R2_name + " > " + fastqFolder + "/"+R2_name + ".gz");
+            script.time("DONE: pigz " + R2_name);
 
             script.cmd("ls -1 "+fastqFolder+"/*.fastq.gz >> " + filesOut);