Changeset 6006


Ignore:
Timestamp:
Sep 18, 2020, 7:46:01 AM (12 months ago)
Author:
Nicklas Nordborg
Message:

References #1266: Run prepDE.py in the StringTie? pipeline

Inlcuded prepDE jobs in the auto-confirmation handling by adding a temporary any-to-any link "gene_count.csv" that points to the current job instead of the actual file. This causes raw bioassays that have been scheduled for processing to not be included in the count anymore andthere is no risk that prepDE is scheduled more than once for the same raw bioassays. If all goes well, the link is replaced with the actual result file, otherwise PrepDEAutoConfirmer will remove the link and it will be possible to try again.

Location:
extensions/net.sf.basedb.reggie/branches/4.27-stable/src/net/sf/basedb/reggie
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/branches/4.27-stable/src/net/sf/basedb/reggie/autoconfirm/AutoConfirmService.java

    r5845 r6006  
    306306      numItems += confirmRawOnlyVariantCalling(dc, manager, "variants-raw.vcf.gz");
    307307     
     308      // Temporary prepDE for existing StringTie items
     309      numItems += confirmPrepDE(dc, manager, "gene_count.csv");
     310     
    308311      // Commit now so that all items are reverted to manual flow just in case
    309312      // Actual auto-confirm and starting of next step is done later
     
    418421  }
    419422
    420  
     423  /**
     424    This will check if there are any '*.csv' AnyToAny links from a RawBioAssay (StringTie)
     425    to a JOB with ERROR status. If so, the link is removed to allowed it to be picked up for
     426    re-processing.
     427    @since 4.27.4
     428  */
     429  private int confirmPrepDE(DbControl dc, AutoConfirmManager manager, String csvName)
     430  {
     431    ItemQuery<RawBioAssay> query = RawBioAssay.getQuery();
     432    query.setIncludes(dc.getSessionControl().getActiveProjectId() != 0 ? Reggie.INCLUDE_IN_CURRENT_PROJECT : Include.ALL);
     433 
     434    // StringTie only
     435    Rawdatatype.STRINGTIE.addFilter(dc, query);
     436   
     437    // Must have '*.csv' linking to a JOB item
     438    query.restrict(AnyToAnyRestriction.exists(csvName, Item.JOB));
     439 
     440    int numItems = 0;
     441    Iterator<RawBioAssay> it = query.iterate(dc);
     442    while (it.hasNext())
     443    {
     444      numItems++;
     445      RawBioAssay stringTie = it.next();
     446      AutoConfirmer<?> ac = new PrepDEAutoConfirmer(stringTie, csvName);
     447      manager.checkRulesAndAdd(dc, ac);
     448    }
     449    return numItems;
     450  }
     451
    421452  /**
    422453    Timer for auto-confirmation
  • extensions/net.sf.basedb.reggie/branches/4.27-stable/src/net/sf/basedb/reggie/grid/PrepDEJobCreator.java

    r6005 r6006  
    144144      }
    145145
     146      // Temporary link to the job so that it is possible to handle failed jobs in the RawOnlyVariantCallingAutoConfirmer
     147      AnyToAny link = AnyToAny.getNewOrExisting(dc, item, "gene_count.csv", prepDEJob, false);
     148      if (!link.isInDatabase()) dc.saveItem(link);
     149     
    146150      script.comment("Running prepDE for " + item.getName());
    147151      script.progress(100 * current / total, "Running prepDE: " + item.getName() + " (" + current + " of " + total+ ")");
     
    153157      String externalGroup = isExternal ? Reggie.getExternalGroup(item.getName()) : null;
    154158      ScriptUtil.setUmaskForItem(dc, lib, externalGroup, script);
    155       script.cmd("mkdir -p ${StringTieFolder}");
     159//      script.cmd("mkdir -p ${StringTieFolder}");
    156160     
    157161      String prepDECmd = prepde_path;
  • extensions/net.sf.basedb.reggie/branches/4.27-stable/src/net/sf/basedb/reggie/servlet/StringTieServlet.java

    r6005 r6006  
    537537           
    538538            // For debugging
    539             //stringTie = stringTie.subList(0, 2);
     539            // stringTie = stringTie.subList(0, 2);
    540540           
    541541            JobDefinition jobDef = prepDE.createPrepDEJob(dc, cluster, stringTie);
     
    552552              jsonMessages.add("Submitted prepDE.py (" + stringTie.size() + " rawbioassays) job to " + cluster.getConnectionInfo().getName() + " with id " + job.getExternalId());
    553553            }
    554             query.setFirstResult(query.getMaxResults()+query.getFirstResult());
     554            // NOTE! query.setFirstResult() is not needed since the previous 500 will get a temporary "gene_count.csv" link
     555            // query.setFirstResult(query.getMaxResults()+query.getFirstResult());
    555556          }
    556557
Note: See TracChangeset for help on using the changeset viewer.