Changeset 3486

Timestamp:

Sep 15, 2015, 1:38:05 PM (8 years ago)

Author:

Nicklas Nordborg

Message:

References #809: Improve performance of demux+merge step

Start gzip with compression level 1 and in the background.

Location:

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie

Files:

• servlet/DemuxMergeServlet.java (modified) (1 diff)
• ssh/ScriptBuilder.java (modified) (5 diffs)

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/DemuxMergeServlet.java

@@ -872 +872 @@
             script.cmd("cat fastq.aligned/" + fragments_name + " >> " + fragmentsOut);
             script.cmd("cp fastq.aligned/"+fragments_name + " " + fastqFolder + "/" + fragments_name);
-            script.cmd("gzip -c fastq.trimmomatic.2/" + R1_name + " > " + fastqFolder + "/"+R1_name + ".gz");
-            script.time("DONE: gzip " + R1_name);
-            script.cmd("gzip -c fastq.trimmomatic.2/" + R2_name + " > " + fastqFolder + "/"+R2_name + ".gz");
-            script.time("DONE: gzip " + R2_name);
+            String pid1 = script.bkgr("gzip -1 -c fastq.trimmomatic.2/" + R1_name + " > " + fastqFolder + "/"+R1_name + ".gz");
+            String pid2 = script.bkgr("gzip -1 -c fastq.trimmomatic.2/" + R2_name + " > " + fastqFolder + "/"+R2_name + ".gz");
+            script.waitForProcess(pid1);
+            script.waitForProcess(pid2);
+            script.time("DONE: gzip " + mergeName);
 
             script.cmd("ls -1 "+fastqFolder+"/*.fastq.gz >> " + filesOut);

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/ssh/ScriptBuilder.java

@@ -1 +1 @@
 package net.sf.basedb.reggie.ssh;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import net.sf.basedb.core.InvalidDataException;
 import net.sf.basedb.reggie.Reggie;
 
@@ -12 +16 @@
   private final StringBuilder script;
   private final String jobFolder;
+  
+  private int nextBgPid;
+  private Map<String, String> bgProcesses;
   
   /**
@@ -95 +102 @@
   
   /**
-    Adds a command that is executed in the background
+    Adds a command that is executed in the background. The return value is the
+    name of a variable containing the process id of the background task. Use this
+    in {@link #waitForProcess(String)} to add a wait statement that also catches the
+    exit code of the background task.
+    @since 3.6
   */
-  @Deprecated
-  public void bkgr(String cmd)
+  public String bkgr(String cmd)
   {
+    if (bgProcesses == null) bgProcesses = new HashMap<String, String>();
+    String pid = "BG"+nextBgPid;
+    nextBgPid++;
+    bgProcesses.put(pid, cmd);
     script.append(cmd).append(" &\n");
+    script.append(pid).append("=$!\n");
+    return pid;
   }
   
@@ -107 +123 @@
     Note that it is only the batch as a whole that is executed in 
     the background, the individual commands are executed in serial.
+    The return value is the name of a variable containing the process 
+    id of the background task. Use this in {@link #waitForProcess(String)} to add 
+    a wait statement that also catches the exit code of the background task.
+    @since 3.6
   */
-  @Deprecated
-  public void bkgr(String... cmds)
+  public String bkgr(String... cmds)
   {
-    script.append("{ ");
+    StringBuilder cmd = new StringBuilder();
+    cmd.append("{ ");
     for (String c : cmds)
     {
-      script.append(c).append("; ");
+      cmd.append(c).append("; ");
     }
-    script.append(" } &\n");
+    cmd.append(" }");
+    return bkgr(cmd.toString());
   }
 
+  /**
+    Wait for the given process to end.
+    @param pid The process id variable name returned by {@link #bkgr(String)}.
+    @since 3.6
+  */
+  public void waitForProcess(String pid)
+  {
+    if (bgProcesses == null || !bgProcesses.containsKey(pid))
+    {
+      throw new InvalidDataException("Unknown background process id: " + pid);
+    }
+    bgProcesses.remove(pid);
+    script.append("wait $").append(pid).append("\n");
+  }
+
+  
   /**
     Get the current length of the script.
@@ -128 +165 @@
   
   /**
-    Get the generated script.
+    Get the generated script. Will throw an exception if {@link #waitForProcess(String)} hasn't 
+    been called for all commands started in the background ({@link #bkgr(String)} or {@link #bkgr(String...)})
   */
   @Override
   public String toString() 
   {
+    if (bgProcesses != null && bgProcesses.size() > 0)
+    {
+      throw new InvalidDataException("The script start background tasks, but does not wait for them: " + bgProcesses);
+    }
     return script.toString();
   }