Changeset 5867


Ignore:
Timestamp:
Mar 18, 2020, 11:44:27 AM (21 months ago)
Author:
Nicklas Nordborg
Message:

References #1231: Add support for sequencing with NovaSeq?

Started to implement a wizard for registering a sample sheet from an external sequencing. A lot of checks are made to try to make sure that only the expected libraries are registered.

The wizard expects that library items have been pre-created and placed on a "External library plate" (it is not possible to do that by a wizard at the moment). The libraries should be without a 'creation date'.

The new wizard will set the 'creation date' to the date found in the sample sheet file (it is possible to manually change this), and associate the library with a barcode. Actual barcode sequences in the sample sheet are verified against the database.

The wizard will then create a single "Pooled library" item for all the libraries as well as a single "Flow cell" item and "Sequencing run". It will try to put in as much information as possible to make it possible for the "auto-confirmation" function to detect when data is available in the run archive and continue with demux and other analysis.

It should also be possible to use the manual "Sequencing ended" wizard.

At the moment, they doesn't work since some information is not yet available. We can probably parse out what we need from the "RunParameters?.xml" file, it just has to be done in a slightly different order.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
3 added
4 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/resources/index.jsp

    r5845 r5867  
    738738          <dt>
    739739            <base:icon image="<%=home+"/images/flowcell.png" %>" />
    740             Clustering and sequencing wizards
     740            Sequencing wizards
    741741          </dt>
    742742          <dd>
     
    763763            </ul>
    764764          </dd>
     765
     766          <dt>
     767            <base:icon image="<%=home+"/images/lorry.png" %>" />
     768            External sequencing wizards
     769          </dt>
     770          <dd>
     771            <ul>
     772     
     773            <li><span class="require-permission" data-role="ExternalSequencing" data-link="sequencing/sequencing_external.jsp?ID=<%=ID%>&pipeline=RNA_SEQ"
     774              >Register external sequencing</span>
     775              <span class="counter" data-counter="external-libs-not-sequenced"
     776                title="Number of libraries sent for external sequencing">∙</span>
     777            </ul>
     778          </dd>
     779
    765780         
    766781          </dl>
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/dao/BioplateType.java

    r5526 r5867  
    106106    new BioplateType("NeoPrep plate", Item.EXTRACT, Subtype.LIBRARY, LockMode.LOCKED_AFTER_ADD, Geometry.EIGHT_BY_TWO, "NeoPrepPlate", 4);
    107107
     108 
     109  /**
     110    The definition of the "External library plate" type. It is locked to extracts
     111    of subtype {@link Subtype#LIBRARY} and uses 8x12 geometry. TODO -- different geometry??
     112    @since 2.12
     113  */
     114  public static final BioplateType EXTERNAL_LIBRARY =
     115    new BioplateType("External library plate", Item.EXTRACT, Subtype.LIBRARY, LockMode.LOCKED_AFTER_ADD, Geometry.EIGHT_BY_TWELVE, "ExternalLibPlate", 4);
     116   
    108117  /**
    109118    The definition of the "MIPs plate" type. It is locked to extracts
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/InstallServlet.java

    r5865 r5867  
    362362        jsonChecks.add(checkBioplateType(dc, BioplateType.CDNA, createIfMissing));
    363363        jsonChecks.add(checkBioplateType(dc, BioplateType.LIBRARY, createIfMissing));
     364        jsonChecks.add(checkBioplateType(dc, BioplateType.EXTERNAL_LIBRARY, createIfMissing));
    364365        jsonChecks.add(checkBioplateType(dc, BioplateType.NEOPREP, createIfMissing));
    365366        jsonChecks.add(checkBioplateType(dc, BioplateType.MIPS, createIfMissing));
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/SequencingRunServlet.java

    r5857 r5867  
    44import java.util.ArrayList;
    55import java.util.Date;
     6import java.util.HashMap;
    67import java.util.HashSet;
    78import java.util.List;
     9import java.util.Map;
    810import java.util.Set;
    911import java.util.regex.Matcher;
     
    2123import net.sf.basedb.core.AnyToAny;
    2224import net.sf.basedb.core.BioMaterialEvent;
     25import net.sf.basedb.core.BioPlate;
     26import net.sf.basedb.core.BioPlateType;
     27import net.sf.basedb.core.BioWell;
    2328import net.sf.basedb.core.BooleanParameterType;
    2429import net.sf.basedb.core.ItemList;
     
    2833import net.sf.basedb.core.Hardware;
    2934import net.sf.basedb.core.IntegerParameterType;
     35import net.sf.basedb.core.Item;
    3036import net.sf.basedb.core.ItemNotFoundException;
    3137import net.sf.basedb.core.ItemQuery;
     38import net.sf.basedb.core.ItemSubtype;
    3239import net.sf.basedb.core.Job;
    3340import net.sf.basedb.core.PhysicalBioAssay;
    3441import net.sf.basedb.core.Protocol;
    3542import net.sf.basedb.core.SessionControl;
     43import net.sf.basedb.core.Tag;
     44import net.sf.basedb.core.Type;
    3645import net.sf.basedb.core.query.Annotations;
    3746import net.sf.basedb.core.query.Expressions;
     
    6473import net.sf.basedb.reggie.extensions.ReggieSignalHandlerFactory;
    6574import net.sf.basedb.reggie.grid.ScriptUtil;
     75import net.sf.basedb.util.EqualsHelper;
    6676import net.sf.basedb.util.Values;
    6777import net.sf.basedb.util.XmlUtil2;
     
    674684        Annotationtype.SEQUENCING_RESULT.setAnnotationValue(dc, flowCell, outcome);
    675685        jsonMessages.add(seqRun.getName() + " confirmed with status: " + outcome);
     686       
     687        dc.commit();
     688      }
     689     
     690      else if ("CheckExternalSequencing".equals(cmd))
     691      {
     692        dc = sc.newDbControl();
     693
     694        ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.SECONDARY_ANALYSIS, ReggieRole.ADMINISTRATOR);
     695
     696        JSONObject jsonReq = JsonUtil.parseRequest(req);
     697        JSONArray jsonLibs = (JSONArray)jsonReq.get("libraries");
     698       
     699        ItemQuery<Extract> findLib = Extract.getQuery();
     700        findLib.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     701        Subtype.LIBRARY.addFilter(dc, findLib);
     702        findLib.restrict(Restrictions.eq(Hql.property("name"), Expressions.parameter("lib")));
     703       
     704        ItemQuery<Tag> findBarcode = Tag.getQuery();
     705        findBarcode.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     706        Subtype.BARCODE.addFilter(dc, findBarcode);
     707        findBarcode.restrict(Restrictions.eq(Hql.property("name"), Expressions.parameter("barcode")));
     708       
     709        JSONArray jsonErrors = new JSONArray();
     710       
     711        Map<String, Number> usedItems = new HashMap<>();
     712        Set<BioPlate> libPlates = new HashSet<>();
     713        Set<Integer> libsInSampleSheet = new HashSet<>();
     714        for (int libNo = 0; libNo < jsonLibs.size(); libNo++)
     715        {
     716          JSONObject jsonLib = (JSONObject)jsonLibs.get(libNo);
     717          Number lineNo = (Number)jsonLib.get("lineNo");
     718          String libName = (String)jsonLib.get("name");
     719          String barcodeName = (String)jsonLib.get("barcode");
     720         
     721          if (usedItems.containsKey(libName))
     722          {
     723            jsonErrors.add("[Error] Line " + lineNo + ": Duplicate library '" + libName + "' also used on line " + usedItems.get(libName));
     724          }
     725          else
     726          {
     727            findLib.setParameter("lib", libName, Type.STRING);
     728            List<Extract> libs = findLib.list(dc);
     729            if (libs.size() == 1)
     730            {
     731              Extract lib = libs.get(0);
     732              jsonLib.put("id", lib.getId());
     733              libsInSampleSheet.add(lib.getId());
     734             
     735              if (lib.getCreationEvent().getEventDate() != null)
     736              {
     737                jsonErrors.add("[Error] Line " + lineNo + ": Library '" + libName + "' is already registered for sequencing");
     738              }
     739             
     740              BioWell well = lib.getBioWell();
     741              if (well != null)
     742              {
     743                libPlates.add(well.getPlate());
     744                // Don't need this now, but maybe in the future
     745                //jsonLib.put("bioWell", JsonUtil.getBioWellAsJSON(well, true));
     746              }
     747             
     748            }
     749            else
     750            {
     751              if (libs.size() == 0)
     752              {
     753                jsonErrors.add("[Error] Line " + lineNo + ": Can't find library '" + libName + "'");
     754              }
     755              else
     756              {
     757                jsonErrors.add("[Error] Line " + lineNo + ": Found " + libs.size() + " libraries with name '" + libName + "'");
     758              }
     759            }
     760          }
     761         
     762          if (usedItems.containsKey(barcodeName))
     763          {
     764            jsonErrors.add("[Error] Line " + lineNo + ": Duplicate barcode '" + barcodeName + "' also used on line " + usedItems.get(barcodeName));
     765          }
     766          else
     767          {
     768            usedItems.put(barcodeName, lineNo);
     769            findBarcode.setParameter("barcode", barcodeName, Type.STRING);
     770            List<Tag> barcodes = findBarcode.list(dc);
     771            if (barcodes.size() == 1)
     772            {
     773              Tag barcode = barcodes.get(0);
     774              jsonLib.put("barcodeId", barcode.getId());
     775
     776              String index1 = (String)jsonLib.get("index1");
     777              String index2 = (String)jsonLib.get("index2");
     778              String seq1 = (String)Annotationtype.BARCODE_SEQUENCE.getAnnotationValue(dc, barcode);
     779              String seq2 = (String)Annotationtype.BARCODE_SEQUENCE_2.getAnnotationValue(dc, barcode);
     780              if (!EqualsHelper.equals(index1, seq1))
     781              {
     782                jsonErrors.add("[Error] Line " + lineNo + ": Barcode sequence mismatch for " + barcodeName + ": " + index1 + "!="+seq1);
     783              }
     784              if (!EqualsHelper.equals(index2, seq2))
     785              {
     786                jsonErrors.add("[Error] Line " + lineNo + ": Barcode sequence mismatch for " + barcodeName + ": " + index2 + "!="+seq2);
     787              }
     788            }
     789            else
     790            {
     791              if (barcodes.size() == 0)
     792              {
     793                jsonErrors.add("[Error] Line " + lineNo + ": Can't find barcode '" + barcodeName + "'");
     794              }
     795              else
     796              {
     797                jsonErrors.add("[Error] Line " + lineNo + ": Found " + barcodes.size() + " barcode with name '" + barcodeName + "'");
     798              }
     799            }
     800          }
     801        }
     802       
     803        json.put("libraries", jsonLibs);
     804        json.put("numErrors", jsonErrors.size());
     805        jsonMessages.addAll(jsonErrors);
     806        if (jsonErrors.size() == 0)
     807        {
     808          jsonMessages.add(jsonLibs.size() + " libraries on " + libPlates.size() + " plate(s)");
     809         
     810          // Check if there are remaining libraries on the used lib plates
     811          if (libPlates.size() > 0)
     812          {
     813            ItemQuery<Extract> query = Extract.getQuery();
     814            query.join(Hql.innerJoin("bioWell", "bw"));
     815            query.join(Hql.innerJoin("bw", "bioPlate", "p"));
     816            query.join(Hql.innerJoin("creationEvent", "ce"));
     817            query.restrict(Restrictions.eq(Hql.alias("p"), Hql.entityParameter("libPlate", Item.BIOPLATE)));
     818            query.restrict(Restrictions.eq(Hql.property("ce", "eventDate"), null));
     819            query.restrict(Restrictions.not(
     820              Restrictions.in(Hql.property("id"), Expressions.parameter("libs", libsInSampleSheet, Type.INT))
     821              ));
     822           
     823            BioPlateType externalType = BioplateType.EXTERNAL_LIBRARY.load(dc);
     824            for (BioPlate libPlate : libPlates)
     825            {
     826              if (!externalType.equals(libPlate.getBioPlateType()))
     827              {
     828                jsonMessages.add("[Warning] '" + libPlate.getName() + "' is not an 'External library plate'");
     829              }
     830              query.setEntityParameter("libPlate", libPlate);
     831              long remainingLibs = query.count(dc);
     832              if (remainingLibs > 0)
     833              {
     834                jsonMessages.add("[Warning] On '" + libPlate.getName() + "' there are " + remainingLibs + " remaining libraries not in the sample sheet");
     835              }
     836            }
     837          }
     838        }
     839      }
     840     
     841      else if ("RegisterExternalSequencing".equals(cmd))
     842      {
     843        dc = sc.newDbControl();
     844
     845        ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.SECONDARY_ANALYSIS, ReggieRole.ADMINISTRATOR);
     846
     847        JSONObject jsonReq = JsonUtil.parseRequest(req);
     848        JSONArray jsonLibs = (JSONArray)jsonReq.get("libraries");
     849        JSONObject jsonAutoAnalyze = (JSONObject)jsonReq.get("autoAnalyze");
     850        String flowCellId = (String)jsonReq.get("flowCellId");
     851        String flowCellType = (String)jsonReq.get("flowCellType");
     852        Date startDate = Reggie.CONVERTER_STRING_TO_DATE.convert((String)jsonReq.get("startDate"));
     853        Number numLanes = (Number)jsonReq.get("numLanes");
     854       
     855        Extract pool = Extract.getNew(dc);
     856        pool.setItemSubtype(Subtype.POOLED_LIBRARY.load(dc));
     857        Pipeline.RNA_SEQ.setAnnotation(dc, pool);
     858        pool.setName(PooledLibrary.getNextNames(dc, 1).get(0));
     859        dc.saveItem(pool);
     860        jsonMessages.add("Created " + pool.getName() + " with " + jsonLibs.size() + " libraries");
     861     
     862        BioMaterialEvent poolEvent = pool.getCreationEvent();
     863        poolEvent.setEventDate(startDate);
     864        poolEvent.setProtocol(null); // To avoid that a 'project default' is used
     865        poolEvent.setHardware(null);
     866       
     867        Set<BioPlate> libPlates = new HashSet<>();
     868        for (int libNo = 0; libNo < jsonLibs.size(); libNo++)
     869        {
     870          JSONObject jsonLib = (JSONObject)jsonLibs.get(libNo);
     871          Number libId = (Number)jsonLib.get("id");
     872          Number barcodeId = (Number)jsonLib.get("barcodeId");
     873         
     874          Extract lib = Extract.getById(dc, libId.intValue());
     875          Tag barcode = Tag.getById(dc, barcodeId.intValue());
     876          BioWell well = lib.getBioWell();
     877          if (well != null)
     878          {
     879            BioPlate plate = well.getPlate();
     880            libPlates.add(plate);
     881          }
     882         
     883          // Update library properties
     884          lib.setTag(barcode);
     885          lib.getCreationEvent().setEventDate(startDate);
     886         
     887          // Link with pool
     888          poolEvent.addSource(lib);
     889        }
     890       
     891        // Link pool with lib plates
     892        int plateNo = 0;
     893        for (BioPlate plate : libPlates)
     894        {
     895          String linkSuffix = plateNo == 0 ? "" : "."+(plateNo+1);
     896          AnyToAny libPlateLink = AnyToAny.getNew(dc, pool, plate, "LibPlate"+linkSuffix, true);
     897          dc.saveItem(libPlateLink);
     898          plateNo++;
     899        }
     900       
     901        // Create FlowCell and SequencingRun
     902        PhysicalBioAssay flowCell = PhysicalBioAssay.getNew(dc);
     903        flowCell.setItemSubtype(Subtype.FLOW_CELL.load(dc));
     904        Pipeline.RNA_SEQ.setAnnotation(dc, flowCell);
     905        flowCell.setName(FlowCell.getNextNames(dc, 1).get(0));
     906        flowCell.setSize(numLanes.intValue());
     907        Annotationtype.FLOWCELL_ID.setAnnotationValue(dc, flowCell, flowCellId);
     908        Annotationtype.FLOWCELL_TYPE.setAnnotationValue(dc, flowCell, flowCellType);
     909        dc.saveItem(flowCell);
     910        jsonMessages.add("Created " + flowCell.getName() + " with " + numLanes + " lanes");
     911
     912        // SequencingCycles ??
     913       
     914        // Create pool aliquots
     915        BioMaterialEvent flowCellEvent = flowCell.getCreationEvent();
     916        flowCellEvent.setEventDate(startDate);
     917        flowCellEvent.setHardware(null); // To prevent 'project default'
     918        flowCellEvent.setProtocol(null);
     919        ItemSubtype poolAliquotType = Subtype.POOLED_LIBRARY_ALIQUOT.load(dc);
     920        for (int laneNo = 1; laneNo <= flowCell.getSize(); laneNo++)
     921        {
     922          Extract poolA = Extract.getNew(dc);
     923          poolA.setItemSubtype(poolAliquotType);
     924          poolA.setName(pool.getName()+"."+laneNo);
     925          Pipeline.RNA_SEQ.setAnnotation(dc, poolA);
     926          poolA.getCreationEvent().setSource(pool);
     927          flowCellEvent.addSource(poolA).setPosition(laneNo);
     928          dc.saveItem(poolA);
     929        }
     930       
     931        Job sequencingJob = null;
     932        String seqRunName = SequencingRun.getNextName(dc);
     933
     934        if (jsonAutoAnalyze != null)
     935        {
     936          boolean debug = Boolean.TRUE.equals(jsonAutoAnalyze.get("debug"));
     937          Number priority = (Number)jsonAutoAnalyze.get("priority");
     938          String clusterId = (String)jsonAutoAnalyze.get("cluster");
     939         
     940          OpenGridCluster cluster = OpenGridService.getInstance().getClusterById(dc, clusterId);
     941          if (cluster == null)
     942          {
     943            throw new ItemNotFoundException("OpenGridCluster[" + clusterId + "]");
     944          }
     945         
     946          sequencingJob = Job.getNew(dc, null, null, null);
     947          sequencingJob.setItemSubtype(Subtype.SEQUENCING_RUN_JOB.get(dc));
     948          sequencingJob.setPluginVersion("reggie-"+Reggie.VERSION);
     949          sequencingJob.setSendMessage(Values.getBoolean(sc.getUserClientSetting("plugins.sendmessage"), false));
     950          sequencingJob.setName(seqRunName + (debug ? " (debug)" : ""));
     951          if (debug)
     952          {
     953            sequencingJob.setParameterValue("debug", new BooleanParameterType(), debug);
     954          }
     955          if (priority != null)
     956          {
     957            sequencingJob.setParameterValue("priority", new IntegerParameterType(), priority.intValue());
     958          }
     959
     960          sequencingJob.setScheduled(cluster.getId(), null);
     961          sequencingJob.setExternalId(flowCellId);
     962         
     963          String signalURI = ReggieSignalHandlerFactory.getSignalUri(flowCellType, flowCellId);
     964          if (signalURI != null)
     965          {
     966            sequencingJob.setSignalTransporter(ExtensionSignalTransporter.class, signalURI);
     967          }
     968         
     969          dc.saveItem(sequencingJob);
     970        }
     971       
     972        DerivedBioAssay seqRun = DerivedBioAssay.getNew(dc, flowCell, sequencingJob);
     973        seqRun.setItemSubtype(Subtype.SEQUENCING_RUN.load(dc));
     974        Pipeline.RNA_SEQ.setAnnotation(dc, seqRun);
     975        seqRun.setName(seqRunName);
     976        Annotationtype.SEQUENCING_START.setAnnotationValue(dc, seqRun, startDate);
     977        seqRun.setHardware(null); // To prevent a 'project default'
     978        seqRun.setProtocol(null);
     979        dc.saveItem(seqRun);
     980       
     981        jsonMessages.add("Created " + seqRun.getName());
    676982       
    677983        dc.commit();
Note: See TracChangeset for help on using the changeset viewer.