Changeset 5827

Timestamp:

Feb 19, 2020, 9:25:09 AM (3 years ago)

Author:

Nicklas Nordborg

Message:

References #1218: Implement MIPs alignment

Added Trimmomatic steps to the script. Logging and error handling is not fully implemented yet.

Location:

extensions/net.sf.basedb.reggie/trunk

Files:

• config/reggie-config.xml (modified) (1 diff)
• src/net/sf/basedb/reggie/dao/Datafiletype.java (modified) (2 diffs)
• src/net/sf/basedb/reggie/grid/MipsAlignJobCreator.java (modified) (8 diffs)

extensions/net.sf.basedb.reggie/trunk/config/reggie-config.xml

@@ -325 +325 @@
         <parallel-environment>smp 8-16</parallel-environment>
         
+        <!-- Options for Trimmomatic -->
+        <trimmomatic>
+          <!-- Optional path to Trimmomatic, if not specified the default in the 'programs' section is used -->
+          <path>/usr/local/packages/trimmomatic/0.39/trimmomatic.jar</path>
+          <!-- The first step should filter Illumina adapters-->
+          <step-1>ILLUMINACLIP:adapter.fa:3:12:7:1:true MINLEN:30</step-1>
+          <!-- The second step is for all other filters -->
+          <step-2>MAXINFO:30:0.25 MINLEN:30</step-2>
+        </trimmomatic>
+        
       </align-mips>
       

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Datafiletype.java

@@ -2 +2 @@
 
 import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
@@ -266 +268 @@
     return member == null ? null : member.getFile();
   }
+  
+  /**
+    Get all files of this type that are attached to the given
+    item.
+    @since 4.26
+  */
+  public List<File> getAllFiles(DbControl dc, FileStoreEnabled item)
+  {
+    if (!item.hasFileSet()) return Collections.emptyList();
+    ItemQuery<FileSetMember> query = item.getFileSet().getMembers(get(dc));
+    List<File> files = new ArrayList<>();
+    for (FileSetMember member : query.list(dc))
+    {
+      files.add(member.getFile());
+    }
+    return files;
+  }
 
   /**

extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/grid/MipsAlignJobCreator.java

@@ -3 +3 @@
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
 import org.slf4j.Logger;
@@ -9 +11 @@
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.DerivedBioAssay;
+import net.sf.basedb.core.File;
 import net.sf.basedb.core.ItemList;
 import net.sf.basedb.core.ItemNotFoundException;
@@ -30 +33 @@
 import net.sf.basedb.reggie.dao.Annotationtype;
 import net.sf.basedb.reggie.dao.BiomaterialList;
+import net.sf.basedb.reggie.dao.Datafiletype;
 import net.sf.basedb.reggie.dao.DoNotUse;
 import net.sf.basedb.reggie.dao.Library;
@@ -132 +136 @@
     // Paths to programs used (bowtie , hisat, picards, and more...)
     String pipeline_scripts_path = cfg.getRequiredConfig("programs/pipeline-scripts/path", null);
+    String java_path = cfg.getRequiredConfig("programs/java/path", alignParameterSet);
+    String trimmomatic_path = cfg.getConfig("align-mips/trimmomatic/path", alignParameterSet, null);
+    if (trimmomatic_path == null)
+    {
+      trimmomatic_path = cfg.getRequiredConfig("programs/trimmomatic/path", alignParameterSet);
+    }
 
     // Options for the programs
     String align_parallelEnvironment = cfg.getConfig("align-mips/parallel-environment", alignParameterSet, "smp 8-");
+    String align_trimmomaticOptions1 = cfg.getRequiredConfig("align-mips/trimmomatic/step-1", alignParameterSet);
+    String align_trimmomaticOptions2 = cfg.getRequiredConfig("align-mips/trimmomatic/step-2", alignParameterSet);
 
     // Load common items
@@ -166 +178 @@
       Sample specimen =  (Sample)lib.findSingleParent(dc, Subtype.SPECIMEN);
       String fastQFolder = (String)Annotationtype.DATA_FILES_FOLDER.getAnnotationValue(dc, merged);
+      
+      // Get all FASTQ files
+      List<File> fastqFiles = Datafiletype.FASTQ.getAllFiles(dc, merged);
+      Set<String> fastqPrefix = new TreeSet<>();
+      String R1_suffix = "_R1.fastq.gz";
+      for (File f : fastqFiles)
+      {
+        String name = ScriptUtil.checkValidFilename(f.getName());
+        if (name.endsWith(R1_suffix))
+        {
+          String prefix = name.substring(0, name.length()-R1_suffix.length());
+          fastqPrefix.add(prefix);
+        }
+      }
       
       /*
@@ -228 +254 @@
       script.comment("Setting up scripting environment and copying script to tmp folder");
       script.cmd("ScriptDir=" + pipeline_scripts_path);
+      script.cmd("JAVA="+java_path);
+      script.cmd("TrimmomaticJAR="+trimmomatic_path);
       script.cmd("ArchiveRoot="+archiveRoot);
       script.cmd("FastqFolder=${ArchiveRoot}"+fastQFolder);
+      script.cmd("FastqPrefix=( \\");
+      script.cmd("  " + Values.getString(fastqPrefix, " \\\n  ", true) + " \\");
+      script.cmd("  )");
+      
       script.newLine();
   
@@ -243 +275 @@
       script.comment("Move to the temporary working directory and copy the pipeline scripts");
       script.cmd("cd ${TMPDIR}");
+      script.cmd("mkdir fastq");
+      script.cmd("mkdir trimmomatic.1");
+      script.cmd("mkdir trimmomatic.2");
       script.cmd("cp ${ScriptDir}/stdwrap.sh .");
-      script.newLine();
-
+      script.cmd("cp ${ScriptDir}/mips_adapters.sh .");
+      script.newLine();
+
+      script.comment("Setup adapters in a separate script");
+      script.cmd(". ./mips_adapters.sh");
+      script.newLine();
+      
       script.comment("Copy FASTQ files to tmp folder");
       script.progress(10, "Copying FASTQ files");
@@ -252 +292 @@
       script.cmd("exit 1");
       script.cmd("fi");
-      script.cmd("mkdir fastq");
       script.cmd("cp ${FastqFolder}/*.fastq.gz fastq");
       script.newLine();
 
-      /*
-      script.comment("Find FASTQ files");
-      script.cmd("FASTQ1=`find fastq -name \"*_R1.fastq.gz\" -print -quit 2> /dev/null`");
-      script.cmd("FASTQ2=`find fastq -name \"*_R2.fastq.gz\" -print -quit 2> /dev/null`");
-      script.newLine();
-      */
-
-
+      script.comment("Run Trimmomatic");
+      script.cmd("for prefix in ${FastqPrefix[@]} ; do");
+      
+      String trimCmd1 = "./stdwrap.sh ${JAVA} -jar ${TrimmomaticJAR} PE";
+      trimCmd1 += " -threads ${NumThreads}";
+      trimCmd1 += " -phred33 -trimlog trimmomatic.1/${prefix}.log";
+      trimCmd1 += " fastq/${prefix}_R1.fastq.gz";
+      trimCmd1 += " fastq/${prefix}_R2.fastq.gz";
+      trimCmd1 += " trimmomatic.1/${prefix}_R1.fastq";
+      trimCmd1 += " trimmomatic.1/un_${prefix}_R1.fastq";
+      trimCmd1 += " trimmomatic.1/${prefix}_R2.fastq";
+      trimCmd1 += " trimmomatic.1/un_${prefix}_R2.fastq";
+      if (align_trimmomaticOptions1 != null)
+      {
+        trimCmd1 += " " + align_trimmomaticOptions1;
+      }
+      trimCmd1 += " >> trimmomatic.1/trimmomatic.out";
+      
+      String trimCmd2 = "./stdwrap.sh ${JAVA} -jar ${TrimmomaticJAR} PE";
+      trimCmd2 += " -threads ${NumThreads}";
+      trimCmd2 += " -phred33 -trimlog trimmomatic.2/${prefix}.log";
+      trimCmd2 += " trimmomatic.1/${prefix}_R1.fastq";
+      trimCmd2 += " trimmomatic.1/${prefix}_R2.fastq";
+      trimCmd2 += " trimmomatic.2/${prefix}_R1.fastq";
+      trimCmd2 += " trimmomatic.2/un_${prefix}_R1.fastq";
+      trimCmd2 += " trimmomatic.2/${prefix}_R2.fastq";
+      trimCmd2 += " trimmomatic.2/un_${prefix}_R2.fastq";
+      if (align_trimmomaticOptions2 != null)
+      {
+        trimCmd2 += " " + align_trimmomaticOptions2;
+      }
+      trimCmd2 += " >> trimmomatic.2/trimmomatic.out";
+      script.cmd("   " + trimCmd1);
+      script.cmd("   " + trimCmd2);   
+      script.cmd("done");
+      
       /*
       script.progress(95, "Copying result files to project archive");