Changeset 3907


Ignore:
Timestamp:
Apr 29, 2016, 2:24:53 PM (7 years ago)
Author:
Nicklas Nordborg
Message:

References #883: Add support for progress reporting to the Reggie wizard API

Added progress reporter to the 'Sample processing statistics' wizard. This requried a bit of refactoring of the code to make it iterate only once instead of multiple times over the items. Sevaral lists objects that was only used for lookup was converted to set objects instead.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/resources/reports/scanbquartermonthreportgenerator.js

    r2842 r3907  
    124124    url += '&cvariant='+encodeURIComponent(frm.chartvariant.value);
    125125   
    126     Wizard.showLoadingAnimation('Generating plots...');
     126    Wizard.showLoadingAnimation('Generating plots...', 'scanbquartermonthreport-progress');
    127127    request = Wizard.asyncJsonRequest(url, report.onPlotGenerated);
    128128  }
  • extensions/net.sf.basedb.reggie/trunk/resources/reports/scanbquartermonthreportgenerator.jsp

    r2611 r3907  
    151151   
    152152    <div id="wizard-status"></div>
     153    <div id="wizard-progress"></div>
    153154
    154155    <table class="navigation" id="navigation">
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/Reggie.java

    r3853 r3907  
    3939import net.sf.basedb.reggie.dao.Rna;
    4040import net.sf.basedb.util.FileUtil;
     41import net.sf.basedb.util.Values;
    4142import net.sf.basedb.util.XmlUtil2;
    4243
     
    195196    return name.toLowerCase().replaceAll("[^a-z0-9_]", "");
    196197  }
     198 
     199  public static String formatCount(long count)
     200  {
     201    if (count > 1000000)
     202    {
     203      return Values.formatNumber(count / 1000000f, 1, "M");
     204    }
     205    else if (count > 1000)
     206    {
     207      return Values.formatNumber(count / 1000f, 1, "k");
     208    }
     209    return Long.toString(count);
     210  }
     211
    197212
    198213  /**
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/ScanBQuarterMonthReportServlet.java

    r3571 r3907  
    66import java.util.Date;
    77import java.util.HashMap;
     8import java.util.HashSet;
     9import java.util.Iterator;
    810import java.util.ArrayList;
    911import java.util.List;
     12import java.util.Set;
    1013
    1114import javax.servlet.ServletException;
     
    2326import net.sf.basedb.core.Sample;
    2427import net.sf.basedb.core.SessionControl;
     28import net.sf.basedb.core.SimpleProgressReporter;
    2529import net.sf.basedb.core.query.Annotations;
    2630import net.sf.basedb.core.query.Expressions;
     
    108112
    109113  private int minItemsForStatisticsCalculation = 5;
    110   private List<Sample> allSamples;
     114  private Set<Sample> allSamples = new HashSet<Sample>();
    111115  private List<Sample> sampleHistologyList = new ArrayList<Sample>();
    112   private List<Sample> sampleRnaList = new ArrayList<Sample>();
    113   private List<Sample> sampleDnaList = new ArrayList<Sample>();
    114116  private List<Sample> sampleNegativeMinToRnaLaterList = new ArrayList<Sample>();
    115117  private List<Sample> sampleZeroMinToRnaLaterList = new ArrayList<Sample>();
    116   private List<Extract> allExtracts;
    117   private List<Extract> extractLysateList = new ArrayList<Extract>();
     118  private Set<Extract> allLysates = new HashSet<Extract>();
    118119  private List<Extract> extractDnaList = new ArrayList<Extract>();
    119120  private List<Extract> extractRnaList = new ArrayList<Extract>();
    120   private List<Extract> extractRnaQcRawList = new ArrayList<Extract>();
    121121  private List<Extract> extractRnaQcList = new ArrayList<Extract>();
    122   private List<Integer> rnaQcGrandParentSampleIdList = new ArrayList<Integer>();
     122  private Set<Integer> rnaQcGrandParentSampleIds = new HashSet<Integer>();
    123123
    124124  private HashMap<Integer, Date> sampleIdQiaCubeDateHashMap = new HashMap<Integer, Date>();
     
    189189      }
    190190      else if ("scanbquartermonthreport".equals(cmd))
    191       {       
     191      {
     192        SimpleProgressReporter progress = new SimpleProgressReporter(null);
     193        sc.setSessionSetting("scanbquartermonthreport-progress", progress);
     194        progress.display(1, "Loading samples...");
     195       
    192196        String startDateParameter = Values.getString(req.getParameter("fdate"), null);
    193197        String endDateParameter = Values.getString(req.getParameter("tdate"), null);
     
    234238        String chartVariant = Values.getString(req.getParameter("cvariant"), originalQuantityTissueChart);
    235239
    236         json = createScanBQuarterMonthReport(dc, json, startDate, endDate, viewType, chartVariant, site);
     240        json = createScanBQuarterMonthReport(dc, json, startDate, endDate, viewType, chartVariant, site, progress);
     241        progress.display(100, "Done");
    237242      }
    238243    }
     
    248253    {
    249254      if (dc != null) dc.close();
     255      if (sc != null) sc.setSessionSetting("scanbquartermonthreport-progress", null);
    250256      json.writeJSONString(resp.getWriter());
    251257    }
     
    253259
    254260  @SuppressWarnings("unchecked")
    255   private JSONObject createScanBQuarterMonthReport(DbControl dc, JSONObject json, Date startDate, Date endDate, String viewType, String chartVariant, Site site)
     261  private JSONObject createScanBQuarterMonthReport(DbControl dc, JSONObject json, Date startDate, Date endDate, String viewType, String chartVariant, Site site, SimpleProgressReporter progress)
    256262    throws ServletException, IOException
    257263  {
     
    281287    sampleQuery.order(Orders.asc(Hql.property("ce", "eventDate")));
    282288    sampleQuery.setCacheResult(true);
    283     // Perform database query and store result in list
    284     allSamples = sampleQuery.list(dc);
     289
    285290
    286291    // Extract raw list
     
    304309    extractQuery.order(Orders.asc(Hql.property("ce", "eventDate")));
    305310    extractQuery.setCacheResult(true);
    306     // Perform database query and store result in list
    307     allExtracts = extractQuery.list(dc);
    308 
     311   
     312    // Count items
     313    long totalCount = sampleQuery.count(dc);
     314    totalCount += extractQuery.count(dc);
     315    int count = 0;
     316   
    309317    // Store samples that should be processed for report table in list
    310318    ItemSubtype subtypeSpecimen = Subtype.SPECIMEN.load(dc);
     
    321329    SnapshotManager manager = getSnapshotManager();
    322330    // Store samples with original quantity > 0 and with no special biopsy type
    323     for (Sample sample: allSamples)
    324     {
     331    Iterator<Sample> samples = sampleQuery.iterate(dc);
     332    while (samples.hasNext())
     333    {
     334      if (count % 100 == 0)
     335      {
     336        progress.display(5 + (int)((90 * count) / totalCount),
     337          "Processing " + Reggie.formatCount(count) + " of " + Reggie.formatCount(totalCount) + " items");
     338      }
     339      count++;
     340
     341      Sample sample = samples.next();
     342      allSamples.add(sample);
     343     
    325344      ItemSubtype subtype = sample.getItemSubtype();
    326345      if (subtypeHistology.equals(subtype))
     
    362381    }
    363382   
    364     for (Extract extract: allExtracts)
    365     {
     383    Iterator<Extract> extracts = extractQuery.iterate(dc);
     384    while (extracts.hasNext())
     385    {
     386      if (count % 100 == 0)
     387      {
     388        progress.display(5 + (int)((90 * count) / totalCount),
     389          "Processing " + Reggie.formatCount(count) + " of " + Reggie.formatCount(totalCount) + " items");
     390      }
     391      count++;
     392
     393      Extract extract = extracts.next();
    366394      ItemSubtype subtype = extract.getItemSubtype();
    367395      if (subtypeLysate.equals(subtype))
     
    370398        if (allSamples.contains(specimen))
    371399        {
    372           extractLysateList.add(extract);
     400          allLysates.add(extract);
    373401          // Update tissue quantity used (several extracts may be made from the same parent sample)
    374402          Float quantityTissueUsed = extract.getCreationEvent().getUsedQuantity(specimen);
     
    376404        }
    377405      }
    378     }
    379     for (Extract extract: allExtracts)
    380     {
    381       ItemSubtype subtype = extract.getItemSubtype();
    382       if (subtypeDna.equals(subtype))
     406      else if (subtypeDna.equals(subtype))
    383407      {
    384408        BioMaterial dnaParent = extract.getParent();
    385         if (extractLysateList.contains(dnaParent))
     409        if (allLysates.contains(dnaParent))
    386410        {
    387411          Extract lysate = (Extract)dnaParent;
     
    390414          extractIdQiaCubeDateHashMap.put(extract.getId(), date);
    391415          Sample sample = (Sample) lysate.getParent();
    392           sampleDnaList.add(sample);
    393416          sampleIdQiaCubeDateHashMap.put(sample.getId(), date);
    394417          // Store sample id for DNA yield calculation
     
    399422      {
    400423        BioMaterial rnaParent = extract.getParent();
    401         if (extractLysateList.contains(rnaParent))
     424        if (allLysates.contains(rnaParent))
    402425        {
    403426          Extract lysate = (Extract)rnaParent;
     
    406429          extractIdQiaCubeDateHashMap.put(extract.getId(), date);
    407430          Sample sample = (Sample) lysate.getParent();
    408           sampleRnaList.add(sample);
    409431          sampleIdQiaCubeDateHashMap.put(sample.getId(), date);
    410432          // Store sample id for RNA yield calculation
     
    414436      else if (subtypeRnaQc.equals(subtype))
    415437      {
    416         extractRnaQcRawList.add(extract);
    417       }
    418     }
    419     // RNAQC extract are traced back to the grand parent sample they originate from.
    420     // Only one RNAQC value (RQS or RIN) per sample is used.
    421     // If a valid RQS value exists for a sample, it is used, otherwise a valid RIN value.
    422     // For a RQS or RIN value to be valid, it should be > 0 (-100 is used as flag for bad data).
    423     // If several valid RQS or RIN values exist for a sample, the first to occur is used.
    424     //
    425     // Step 1. Find all RNAQC extracts with valid RQS values and list grand parent samples
    426     for (Extract rnaQcExtract: extractRnaQcRawList)
    427     {
    428       Float extractRqs = (Float) Annotationtype.CA_RQS.getAnnotationValue(dc, manager, rnaQcExtract);
    429       // Check if RQS value exists and is > 0 (-100 is used as flag for bad data)
    430       if (extractRqs != null && extractRqs > 0.0f)
    431       {
    432         // Valid RQS value for the RNAQC extract
    433         Extract rnaQcParentExtract = (Extract) rnaQcExtract.getParent();
     438        Extract rnaQcParentExtract = (Extract)extract.getParent();
    434439        Integer sampleId = extractIdSampleIdHashMap.get(rnaQcParentExtract.getId());
    435         // Check if a valid RNAQC values already is stored for the sample
    436         if (sampleId != null && !rnaQcGrandParentSampleIdList.contains(sampleId))
    437         {
    438           extractRnaQcList.add(rnaQcExtract);
    439           extractIdRqsHashMap.put(rnaQcExtract.getId(), extractRqs);
    440           // Use the parent RNA QiaCube date
    441           date = (Date) Annotationtype.QIACUBE_DATE.getAnnotationValue(dc, manager, rnaQcParentExtract);
    442           extractIdQiaCubeDateHashMap.put(rnaQcExtract.getId(), date);
    443           // Add id of grand parent sample to list, in order to check
    444           // that not another RNAQC extract from the same sample is used
    445           rnaQcGrandParentSampleIdList.add(sampleId);
    446         }
    447       }
    448     }
    449     // Step 2. Find all RNAQC extracts with valid RIN values and check
    450     // if another RNAQC value already is stored for parent sample
    451     for (Extract rnaQcExtract: extractRnaQcRawList)
    452     {
    453       Float extractRin = (Float) Annotationtype.BA_RIN.getAnnotationValue(dc, manager, rnaQcExtract);
    454       // Check if RIN value exists and is > 0 (-100 is used as flag for bad data)
    455       if (extractRin != null && extractRin > 0.0f)
    456       {
    457         // Valid RIN value for the RNAQC extract
    458         Extract rnaQcParentExtract = (Extract) rnaQcExtract.getParent();
    459         Integer sampleId = extractIdSampleIdHashMap.get(rnaQcParentExtract.getId());
    460         // Check if a valid RNAQC values already is stored for the sample
    461         if (sampleId != null && !rnaQcGrandParentSampleIdList.contains(sampleId))
    462         {
    463           extractRnaQcList.add(rnaQcExtract);
    464           extractIdRinHashMap.put(rnaQcExtract.getId(), extractRin);
    465           // Use the parent RNA QiaCube date
    466           date = (Date) Annotationtype.QIACUBE_DATE.getAnnotationValue(dc, manager, rnaQcParentExtract);
    467           extractIdQiaCubeDateHashMap.put(rnaQcExtract.getId(), date);
    468           // Add id of grand parent sample to list, in order to check
    469           // that not another RNAQC extract from the same sample is used
    470           rnaQcGrandParentSampleIdList.add(sampleId);
    471         }
    472       }
    473     }
     440       
     441        // RNAQC extract are traced back to the grand parent sample they originate from.
     442        // Only one RNAQC value (RQS or RIN) per sample is used.
     443        // If a valid RQS value exists for a sample, it is used, otherwise a valid RIN value.
     444        // For a RQS or RIN value to be valid, it should be > 0 (-100 is used as flag for bad data).
     445        // If several valid RQS or RIN values exist for a sample, the first to occur is used.
     446       
     447        Float extractRqs = (Float) Annotationtype.CA_RQS.getAnnotationValue(dc, manager, extract);
     448        Float extractRin = (Float) Annotationtype.BA_RIN.getAnnotationValue(dc, manager, extract);
     449
     450        // Check if RQS value exists and is > 0 (-100 is used as flag for bad data)
     451        if (extractRqs != null && extractRqs > 0.0f)
     452        {
     453          // Valid RQS value for the RNAQC extract
     454          // Check if a valid RNAQC values already is stored for the sample
     455          if (sampleId != null && !rnaQcGrandParentSampleIds.contains(sampleId))
     456          {
     457            extractRnaQcList.add(extract);
     458            extractIdRqsHashMap.put(extract.getId(), extractRqs);
     459            // Use the parent RNA QiaCube date
     460            date = (Date) Annotationtype.QIACUBE_DATE.getAnnotationValue(dc, manager, rnaQcParentExtract);
     461            extractIdQiaCubeDateHashMap.put(extract.getId(), date);
     462            // Add id of grand parent sample to list, in order to check
     463            // that not another RNAQC extract from the same sample is used
     464            rnaQcGrandParentSampleIds.add(sampleId);
     465          }
     466        }
     467        else if (extractRin != null && extractRin > 0.0f)
     468        {
     469          // Check if a valid RNAQC values already is stored for the sample
     470          if (sampleId != null && !rnaQcGrandParentSampleIds.contains(sampleId))
     471          {
     472            extractRnaQcList.add(extract);
     473            extractIdRinHashMap.put(extract.getId(), extractRin);
     474            // Use the parent RNA QiaCube date
     475            date = (Date) Annotationtype.QIACUBE_DATE.getAnnotationValue(dc, manager, rnaQcParentExtract);
     476            extractIdQiaCubeDateHashMap.put(extract.getId(), date);
     477            // Add id of grand parent sample to list, in order to check
     478            // that not another RNAQC extract from the same sample is used
     479            rnaQcGrandParentSampleIds.add(sampleId);
     480          }
     481        }
     482      }
     483    }
     484   
     485    progress.display(98, "Processed " + count + " items. Generating plots...");
     486
    474487    // Create list of view types for plots
    475488    List<String> viewTypeList = new ArrayList<String>();
Note: See TracChangeset for help on using the changeset viewer.