Changeset 5845


Ignore:
Timestamp:
Feb 26, 2020, 2:27:54 PM (3 years ago)
Author:
Nicklas Nordborg
Message:

References #1218: Implement MIPs alignment

Added manual confirmation wizard for MIPs alignment. It will display some information and allow registration of successful and failed alignments. Since we don't know what the next step is yet, nothing more is done.

Added auto-confirmation for the MIPs alignment. It will currently only clear the auto-confirm flag and the alignment will appear in the manual confirmation wizard.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
3 added
8 edited

Legend:

Unmodified
Added
Removed

  • extensions/net.sf.basedb.reggie/trunk/resources/index.jsp

    r5826 r5845  
    998998            <dd>
    999999              <ul>
    1000               <li><span class="require-permission" data-role="SecondaryAnalysis" data-link="mipsanalysis/align_start.jsp?ID=<%=ID%>"
     1000              <li><span class="require-permission" data-role="MIPsSecondaryAnalysis" data-link="mipsanalysis/align_start.jsp?ID=<%=ID%>"
    10011001                >Start MIPs alignment</span>
    10021002                <span class="counter" data-counter="merged-sequences-for-mips-alignment"
    10031003                  title="Number of merged sequences waiting for MIPs alignment">∙</span>
    10041004                <span id="mips-align-pipeline-disabled" class="info-warning">auto-processing is disabled</span>
     1005
     1006              <li><span class="require-permission" data-role="MIPsSecondaryAnalysis" data-link="mipsanalysis/align_confirm.jsp?ID=<%=ID%>"
     1007                >Confirm MIPs alignment</span>
     1008                <span class="counter" data-counter="mips-aligned-sequences-not-confirmed"
     1009                  title="Number of MIPs aligned sequences waiting for confirmation">∙</span>
    10051010              </ul>
    10061011            </dd>

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/activity/ActivityDef.java

    r5740 r5845  
    163163
    164164  /**
     165    MIPs alignment manually confirmed. Use {@link #merge(int)}.
     166    @since 4.26
     167  */
     168  public static final ActivityDef MIPS_ALIGNMENT_CONFIRMED = new ActivityDef("mipsalign-confirmed", "Alignment confirmed for 1 MIPs library", "Alignment confirmed for {COUNT} MIPs libraries");
     169  /**
     170    MIPs alignment auto-confirmed. Use {@link #merge(int)}.
     171    @since 4.26
     172  */
     173  public static final ActivityDef MIPS_ALIGNMENT_AUTOCONFIRMED = new ActivityDef("mipsalign-autoconfirmed", "Alignment auto-confirmed for 1 MIPs library", "Alignment auto-confirmed for {COUNT} MIPs libraries");
     174
     175  /**
    165176    HISAT alignment manually confirmed. Use {@link #merge(int)}.
    166177  */

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/autoconfirm/AutoConfirmService.java

    r5724 r5845  
    205205      ItemSubtype alignType = Subtype.ALIGNED_SEQUENCES.load(dc);
    206206      ItemSubtype hisatType = Subtype.HISAT_ALIGN_JOB.load(dc);
     207      ItemSubtype mipsAlignType = Subtype.MIPS_ALIGN_JOB.load(dc);
    207208      RawDataType cufflinksType = Rawdatatype.CUFFLINKS.getRawDataType();
    208209      RawDataType stringTieType = Rawdatatype.STRINGTIE.getRawDataType();
    … …  
    242243          ac = new DemuxAutoConfirmer(dba.getJob());
    243244        }
    244         else if (alignType.equals(itemType) && hisatType.equals(jobType))
    245         {
    246           ac = new HisatAlignAutoConfirmer(dba);
     245        else if (alignType.equals(itemType))
     246        {
     247          if (hisatType.equals(jobType))
     248          {
     249            ac = new HisatAlignAutoConfirmer(dba);
     250          }
     251          else if (mipsAlignType.equals(jobType))
     252          {
     253            ac = new MipsAlignAutoConfirmer(dba);
     254          }
    247255        }
    248256        manager.checkRulesAndAdd(dc, ac);

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/counter/CounterService.java

    r5826 r5845  
    11781178    json.put("merged-sequences-for-mips-alignment", mipsAlignPipeline.getSize());
    11791179   
    1180     // Hisat aligned sequences not confirmed
     1180    // Aligned sequences not confirmed
    11811181    query = DerivedBioAssay.getQuery();
    11821182    Subtype.ALIGNED_SEQUENCES.addFilter(dc, query);
    1183     Pipeline.RNASEQ_HISAT_STRINGTIE.addFilter(dc, query);
    1184     query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
    1185     // The HisatAlign job must be ended
     1183    Pipeline.createFilterParameter(dc, query);
     1184    query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     1185    // The align job must be ended
    11861186    query.join(Hql.innerJoin("job", "jb"));
    1187     Subtype.HISAT_ALIGN_JOB.addFilter(dc, query, "jb");
     1187    Subtype.createFilterParameter(dc, query, "jb", "jobType");
    11881188    query.restrict(Restrictions.neq(Hql.property("jb", "ended"), null));
    11891189    // Must NOT have a ANALYSIS_RESULT annotation
    … …  
    11981198        ));
    11991199    query.setCacheResult(true);
     1200
     1201    // Hisat alignments
     1202    query.setParameter("pipeline", Pipeline.RNASEQ_HISAT_STRINGTIE.getName(), Type.STRING);
     1203    query.setEntityParameter("jobType", Subtype.HISAT_ALIGN_JOB.get(dc));
    12001204    json.put("hisat-aligned-sequences-not-confirmed", query.count(dc));
    12011205    // ---
    1202 
     1206    // MIPs alignments
     1207    query.setParameter("pipeline", Pipeline.MIPS.getName(), Type.STRING);
     1208    query.setEntityParameter("jobType", Subtype.MIPS_ALIGN_JOB.get(dc));
     1209    json.put("mips-aligned-sequences-not-confirmed", query.count(dc));
     1210    // ---
     1211
     1212   
     1213   
    12031214    // Aligned sequences with VCF file that has not been checked
    12041215    query = DerivedBioAssay.getQuery();

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/AlignedSequences.java

    r5705 r5845  
    169169 
    170170  /**
     171    Get the merged sequences item that is an immediate parent item to
     172    this aligned sequences. NOTE! Typically only for the MIPs pipeline.
     173    The RNAseq pipeline has a masked item inbetween.
     174    @since 4.26
     175  */
     176  public MergedSequences getMergedSequences(DbControl dc)
     177  {
     178    ItemQuery<DerivedBioAssay> query = getItem().getParents();
     179    Subtype.MERGED_SEQUENCES.addFilter(dc, query);
     180    query.include(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     181
     182    List<DerivedBioAssay> tmp = query.list(dc);
     183    DerivedBioAssay merged = null;
     184    if (tmp.size() > 1)
     185    {
     186      throw new InvalidDataException(
     187        "More than one parent item was found for " + getName() +
     188        "This wizard can't be used until that is corrected.");
     189    }
     190    else if (tmp.size() == 1)
     191    {
     192      merged = tmp.get(0);
     193    }
     194    return merged == null ? null : MergedSequences.getById(dc, merged.getId());
     195  }
     196
     197 
     198  /**
    171199    Find the next name to give a "Raw bioassay" child item. This assumes that
    172200    all child items are using the naming convention. foo.x, foo.x2, and

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/Subtype.java

    r5826 r5845  
    572572  }
    573573 
     574  /**
     575    Add a filter to the given query to only return items that
     576    belong to a specific subtype. The subtype must be specified
     577    by setting the "parameterName" parameter to the subtype
     578    before executing the query. Use Query.setEntityParameter().
     579    @since 4.26
     580  */
     581  public static void createFilterParameter(DbControl dc, ItemQuery<?> query, String alias, String parameterName)
     582  {
     583    query.restrict(Restrictions.eq(Hql.property(alias, "itemSubtype"), Hql.entityParameter(parameterName, Item.ITEMSUBTYPE)));
     584  }
     585
    574586  private final String name;
    575587  private final String itemSuffix;

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/HisatServlet.java

    r5683 r5845  
    417417          }
    418418        }
    419         numConfirmed++;
     419       
    420420        if (numConfirmed > 0)
    421421        {

  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/MipsServlet.java

    r5826 r5845  
    1919
    2020import net.sf.basedb.core.AnyToAny;
     21import net.sf.basedb.core.BasicItem;
    2122import net.sf.basedb.core.BioMaterialEvent;
    2223import net.sf.basedb.core.BioMaterialEventSource;
    … …  
    4445import net.sf.basedb.core.Software;
    4546import net.sf.basedb.core.Tag;
     47import net.sf.basedb.core.Trashcan;
    4648import net.sf.basedb.core.Type;
    4749import net.sf.basedb.core.query.Annotations;
    … …  
    5860import net.sf.basedb.reggie.activity.ActivityDef;
    5961import net.sf.basedb.reggie.counter.CounterService;
     62import net.sf.basedb.reggie.dao.AlignedSequences;
    6063import net.sf.basedb.reggie.dao.Annotationtype;
    6164import net.sf.basedb.reggie.dao.BiomaterialList;
    … …  
    488491        json.put("mergedSequences", jsonMergedSequences);       
    489492      }
    490 
     493      else if ("GetUnconfirmedAlignedSequences".equals(cmd))
     494      {
     495        dc = sc.newDbControl();
     496        ItemQuery<DerivedBioAssay> query = DerivedBioAssay.getQuery();
     497        query.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     498        Subtype.ALIGNED_SEQUENCES.addFilter(dc, query);
     499        Pipeline.MIPS.addFilter(dc, query);
     500        // The job must be a MIPs alignment job and it must be ended
     501        query.join(Hql.innerJoin("job", "jb"));
     502        Subtype.MIPS_ALIGN_JOB.addFilter(dc, query, "jb");
     503        query.restrict(Restrictions.neq(Hql.property("jb", "ended"), null));
     504        // Must NOT have a ANALYSIS_RESULT annotation
     505        query.join(Annotations.leftJoin(null, Annotationtype.ANALYSIS_RESULT.load(dc), "ar"));
     506        query.restrict(Restrictions.eq(Hql.alias("ar"), null));
     507        // Ignore if AutoProcess==AutoConfirm
     508        query.join(Annotations.leftJoin(null, Annotationtype.AUTO_PROCESSING.load(dc), "ap"));
     509        query.restrict(
     510            Restrictions.or(
     511              Restrictions.neq(Hql.alias("ap"), Expressions.string("AutoConfirm")),
     512              Restrictions.eq(Hql.alias("ap"), null)
     513            ));
     514        // Join LibPlate for sorting
     515        query.join(Hql.innerJoin(null, "extract", "lib", true));
     516        query.join(Hql.innerJoin("lib", "bioWell", "bw", true));
     517        query.join(Hql.innerJoin("bw", "bioPlate", "libPlate", true));
     518        query.order(Orders.asc(Hql.property("libPlate", "id")));
     519        query.order(Orders.asc(Hql.property("name")));
     520        query.setMaxResults(250);
     521       
     522        SnapshotManager manager = new SnapshotManager();
     523        JSONArray jsonAligned = new JSONArray();
     524        List<AlignedSequences> list = AlignedSequences.toList(query.list(dc));
     525        for (AlignedSequences aligned : list)
     526        {
     527          aligned.loadAnnotations(dc, manager, "ALIGNED_PAIRS", Annotationtype.ALIGNED_PAIRS, null);
     528          aligned.loadAnnotations(dc, manager, "ALIGNED_BASES", Annotationtype.ALIGNED_BASES, null);
     529          aligned.loadAnnotations(dc, manager, "READ_PAIRS_EXAMINED", Annotationtype.READ_PAIRS_EXAMINED, null);
     530          aligned.loadAnnotations(dc, manager, "READ_PAIR_DUPLICATES", Annotationtype.READ_PAIR_DUPLICATES, null);
     531          aligned.loadAnnotations(dc, manager, "FRACTION_DUPLICATION", Annotationtype.FRACTION_DUPLICATION, null);
     532          aligned.loadAnnotations(dc, manager, "DataFilesFolder", Annotationtype.DATA_FILES_FOLDER, null);
     533         
     534          // Load job information
     535          Job job = aligned.getItem().getJob();
     536          JSONObject jsonJob = JsonUtil.getJobAsJSON(job);
     537          aligned.setAnnotation("job", jsonJob);
     538
     539          boolean debug = Boolean.TRUE.equals(job.getParameterValue("debug"));
     540          jsonJob.put("debug", debug);
     541         
     542          // Load library information
     543          Library lib = aligned.getLibrary(dc);
     544          Extract dna = (Extract)lib.findSingleParent(dc, Subtype.DNA_NORMALIZED_ALIQUOT);
     545          lib.loadBioPlateLocation();
     546          lib.setAnnotation("MIPSPanel", Annotationtype.MIPS_PANEL.getAnnotationValue(dc, dna));
     547          aligned.setAnnotation("lib", lib.asJSONObject());
     548         
     549          // Load merged information
     550          MergedSequences merged = aligned.getMergedSequences(dc);
     551          merged.loadAnnotations(dc, manager, "PF_READS", Annotationtype.PF_READS, null);
     552          aligned.setAnnotation("merged", merged.asJSONObject());
     553         
     554          jsonAligned.add(aligned.asJSONObject());
     555        }
     556       
     557        json.put("alignedSequences", jsonAligned);
     558       
     559      }
    491560    }
    492561    catch (Throwable t)
    … …  
    10471116        dc.commit();
    10481117      }
     1118      else if ("RegisterAlignment".equals(cmd))
     1119      {
     1120        dc = sc.newDbControl();
     1121
     1122        ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.MIPS_SECONDARY_ANALYSIS, ReggieRole.ADMINISTRATOR);
     1123
     1124        JSONObject jsonReq = JsonUtil.parseRequest(req);
     1125        JSONArray jsonAlignedSequences = (JSONArray)jsonReq.get("alignedSequences");
     1126
     1127        int numConfirmed = 0;
     1128        int numUnconfirmed = 0;
     1129        int numRealign = 0;
     1130        int numFlaggedDna = 0;
     1131        int numDeleted = 0;
     1132       
     1133        ItemList flaggedDna = BiomaterialList.FLAGGED_DNA.load(dc);
     1134        ItemList mipsAlignPipeline = BiomaterialList.MIPS_ALIGN_PIPELINE.load(dc);
     1135//        ItemList nextMipsPipeline = BiomaterialList.MIPS_NEXTSTEP_TODO.load(dc); // TODO
     1136       
     1137        boolean deleteItemsCreatedByFailedJobs = Boolean.TRUE.equals(jsonReq.get("deleteItemsCreatedByFailedJobs"));
     1138        List<BasicItem> toDelete = new ArrayList<BasicItem>();
     1139       
     1140        for (int alignNo = 0; alignNo < jsonAlignedSequences.size(); ++alignNo)
     1141        {
     1142          JSONObject jsonAligned = (JSONObject)jsonAlignedSequences.get(alignNo);
     1143          Number alignedId = (Number)jsonAligned.get("id");
     1144         
     1145          AlignedSequences aligned = AlignedSequences.getById(dc, alignedId.intValue());
     1146          MergedSequences merged = aligned.getMergedSequences(dc);
     1147         
     1148          DerivedBioAssay alignedDBA = aligned.getDerivedBioAssay();
     1149          Job alignJob = alignedDBA.getJob();
     1150          Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, alignedDBA, null);
     1151         
     1152          String flag = (String)(jsonAligned.get("flag"));
     1153          if (flag != null && !Reggie.isExternalItem(aligned.getName()))
     1154          {
     1155            Library lib = Library.get(alignedDBA.getExtract());
     1156            Dna dna = lib.getDna(dc, false);
     1157            Extract d = dna.getItem();
     1158            Annotationtype.FLAG.setAnnotationValue(dc, d, flag);
     1159            flaggedDna.add(d);
     1160            numFlaggedDna++;
     1161          }
     1162         
     1163          String comment = Values.getStringOrNull((String)jsonAligned.get("comment"));
     1164          alignedDBA.setDescription(comment);
     1165         
     1166          boolean reAlign = Boolean.TRUE.equals(jsonAligned.get("realign"));
     1167          boolean startNextStepTODO = Boolean.TRUE.equals(jsonAligned.get("nextStepTODO"));
     1168          boolean deleted = false;
     1169
     1170          if (reAlign)
     1171          {
     1172            // Set ReProcess annotation on merged sequences item
     1173            Annotationtype.AUTO_PROCESSING.setAnnotationValue(dc, merged.getItem(), "ReProcess");
     1174            mipsAlignPipeline.add(merged.getItem());
     1175            numRealign++;
     1176           
     1177            if (deleteItemsCreatedByFailedJobs && alignJob.getStatus() == Job.Status.ERROR)
     1178            {
     1179              // Delete aligned sequences items
     1180              alignedDBA.setRemoved(true);
     1181              toDelete.add(alignedDBA);
     1182              toDelete.addAll(Reggie.removeAttachedFiles(dc, alignedDBA));
     1183              deleted = true;
     1184              numDeleted++;
     1185              if (comment != null)
     1186              {
     1187                merged.getItem().setDescription(comment);
     1188              }
     1189            }
     1190          }
     1191         
     1192          if (!deleted)
     1193          {
     1194            if (startNextStepTODO)
     1195            {
     1196              Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, alignedDBA, AlignedSequences.ALIGN_SUCCESSFUL);
     1197              // nextMipsPipeline.add(alignedDBA);  // TODO
     1198              numConfirmed++;
     1199            }
     1200            else if (reAlign || flag != null)
     1201            {
     1202              Annotationtype.ANALYSIS_RESULT.setAnnotationValue(dc, alignedDBA, AlignedSequences.ALIGN_FAILED);
     1203            }
     1204            else
     1205            {
     1206              numUnconfirmed++;
     1207            }
     1208          }
     1209        }
     1210       
     1211        if (numConfirmed > 0)
     1212        {
     1213          jsonMessages.add(numConfirmed + " libraries confirmed for nextStepTODO"); // TODO
     1214          ActivityDef.MIPS_ALIGNMENT_CONFIRMED.merge(dc, numConfirmed);
     1215        }
     1216        else
     1217        {
     1218          jsonMessages.add("No libraries confirmed as aligned");
     1219        }
     1220       
     1221        if (numRealign > 0)
     1222        {
     1223          jsonMessages.add(numRealign + " libraries flagged for re-alignment");
     1224        }
     1225       
     1226        if (numDeleted > 0)
     1227        {
     1228          jsonMessages.add(numDeleted + " libraries deleted due to failure");
     1229          if (toDelete.size() > numDeleted)
     1230          {
     1231            jsonMessages.add((toDelete.size() - numDeleted) + " linked items (eg. files) deleted");
     1232          }
     1233        }
     1234       
     1235        if (numFlaggedDna > 0)
     1236        {
     1237          jsonMessages.add("Added " + numFlaggedDna + " DNA items to the '" + flaggedDna.getName() + "' list");
     1238        }
     1239       
     1240        if (numUnconfirmed > 0)
     1241        {
     1242          jsonMessages.add(numUnconfirmed + " libraries remain unconfirmed");
     1243        }
     1244       
     1245        dc.commit();
     1246       
     1247        if (toDelete.size() > 0)
     1248        {
     1249          try
     1250          {
     1251            Trashcan.delete(sc, toDelete, false, null);
     1252          }
     1253          catch (RuntimeException ex)
     1254          {
     1255            ex.printStackTrace();
     1256            jsonMessages.add("[Error]Could not delete all items created by failed jobs: " + ex.getMessage());
     1257          }
     1258        }
     1259      }
    10491260      json.put("messages", jsonMessages);
    10501261      CounterService.getInstance().setForceCount();
Note: See TracChangeset for help on using the changeset viewer.