Changeset 5916


Ignore:
Timestamp:
Apr 22, 2020, 1:33:29 PM (17 months ago)
Author:
Nicklas Nordborg
Message:

References #1234: Implement batch importer for followup blood samples

Added a query that checks if a given follow-up blood entry exists based on the RCC ID. So far, only counters are updated and reported back in the progress meter.

Location:
extensions/net.sf.basedb.reggie/trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/resources/personal/import_followup_blood.js

    r5910 r5916  
    100100  blood.submit = function()
    101101  {
     102    var frm = document.forms['reggie'];
     103    var file = frm.sampleFile.files[0];
     104   
     105    var url = '../BloodFollowUpRegistration.servlet?ID='+App.getSessionId();
     106    url += '&cmd=BatchImportFromFile';
     107    url += '&filename='+encodeURIComponent(file.name);
     108    url += '&sheet='+encodeURIComponent(frm.sheet.value);
     109   
     110    Wizard.showLoadingAnimation('Importing ' + Strings.encodeTags(file.name), 'follow-up-import-progress');
     111    Wizard.asyncJsonRequest(url, blood.submissionResults, 'POST', file);
    102112  }
    103113 
  • extensions/net.sf.basedb.reggie/trunk/resources/personal/import_followup_blood.jsp

    r5910 r5916  
    9191
    9292  <div id="wizard-status"></div>
     93  <div id="wizard-progress"></div>
    9394 
    9495  </form>
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/FollowupBloodImporter.java

    r5911 r5916  
    1414
    1515import net.sf.basedb.core.DbControl;
     16import net.sf.basedb.core.ItemQuery;
     17import net.sf.basedb.core.ProgressReporter;
     18import net.sf.basedb.core.Sample;
     19import net.sf.basedb.core.Type;
     20import net.sf.basedb.core.query.Annotations;
     21import net.sf.basedb.core.query.Expressions;
     22import net.sf.basedb.core.query.Hql;
     23import net.sf.basedb.core.query.Restrictions;
     24import net.sf.basedb.reggie.Reggie;
     25import net.sf.basedb.reggie.dao.Annotationtype;
     26import net.sf.basedb.reggie.dao.Blood;
     27import net.sf.basedb.reggie.dao.Subtype;
    1628import net.sf.basedb.util.Values;
    1729import net.sf.basedb.util.excel.XlsxToCsvUtil;
     
    4052  private int numEntries;
    4153  private int numValid;
     54  private int numExisting;
     55  private int numCreated;
    4256 
    4357  public FollowupBloodImporter()
     
    95109  {
    96110    return numValid;
     111  }
     112 
     113  public int getNumCreated()
     114  {
     115    return numCreated;
     116  }
     117 
     118  public int getNumExisting()
     119  {
     120    return numExisting;
    97121  }
    98122 
     
    101125    @param in
    102126  */
    103   public boolean doImport(DbControl dc, InputStream in, String workSheet, boolean validateOnly, String fileName)
     127  public boolean doImport(DbControl dc, InputStream in, String workSheet, boolean validateOnly, String fileName, ProgressReporter progress)
    104128    throws IOException
    105129  {
    106    
    107130    FlatFileParser ffp = getFlatFileParser();
    108131    ffp.setExcelSheet(workSheet);
     
    117140    }
    118141
     142    progress.display(5, "Parsing file " + fileName);
     143   
    119144    LineType headerLine = ffp.parseHeaders();
    120145    int lineNo = ffp.getParsedLines();
     
    140165    Mapper pathNoteMapper = getRequiredMapper(ffp, "OtherPathNote", timeFormat, fileAndLine);
    141166   
    142     // TODO -- more columns
    143167    if (hasError()) return false;
    144168
     
    148172    rccIdPattern = Pattern.compile("\\d{10}(B|C|D)"); // RCC-id should be 10 digits followed by B, C or D
    149173   
     174    List<BloodEntry> validEntries = new ArrayList<>();
    150175    while (ffp.hasMoreData())
    151176    {
     
    173198     
    174199      boolean valid = checkBloodEntry(b);
    175       if (valid) numValid++;
     200      if (valid)
     201      {
     202        numValid++;
     203        validEntries.add(b);
     204      }
     205    }
     206   
     207    if (hasError()) return false;
     208   
     209    if (validateOnly) return true;
     210   
     211    // Import step
     212    int numProcessed = 0;
     213   
     214    ItemQuery<Sample> findBlood = Sample.getQuery();
     215    findBlood.setIncludes(Reggie.INCLUDE_IN_CURRENT_PROJECT);
     216    Subtype.BLOOD.addFilter(dc, findBlood);
     217    findBlood.join(Annotations.innerJoin(Annotationtype.BLOOD_RCCIDNUMBER.get(dc), "rccId"));
     218    findBlood.restrict(Restrictions.eq(Hql.alias("rccId"), Expressions.parameter("rccId", Type.STRING)));
     219   
     220    for (BloodEntry b : validEntries)
     221    {
     222      findBlood.setParameter("rccId", b.rccId, Type.STRING);
     223      List<Sample> blood = findBlood.list(dc);
     224     
     225      if (blood.size() == 0)
     226      {
     227        // TODO -- create Blood item
     228        numCreated++;
     229        String newBloodName = Blood.getNextBloodName(dc, b.caseName);
     230      }
     231      else
     232      {
     233        // TODO -- more than one item??
     234        dc.detachItem(blood.get(0));
     235        numExisting++;
     236      }
     237     
     238      numProcessed++;
     239
     240      if (numProcessed % 100 == 0)
     241      {
     242        progress.display(10 + (80*numProcessed) / numValid, "Processing " + numProcessed + " of " + numValid + " entries; " + numCreated + " new so far...");
     243      }
    176244
    177245    }
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/BloodFollowUpRegistrationServlet.java

    r5911 r5916  
    2323import net.sf.basedb.core.Sample;
    2424import net.sf.basedb.core.SessionControl;
     25import net.sf.basedb.core.SimpleProgressReporter;
    2526import net.sf.basedb.core.query.Hql;
    2627import net.sf.basedb.core.query.Orders;
     
    557558       
    558559        dc = sc.newDbControl();
     560        SimpleProgressReporter progress = new SimpleProgressReporter(null);
    559561        if (!validateOnly)
    560562        {
    561563          ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.PATIENT_CURATOR, ReggieRole.ADMINISTRATOR);
     564          sc.setSessionSetting("follow-up-import-progress", progress);
    562565        }
    563566       
    564567        FollowupBloodImporter importer = new FollowupBloodImporter();
    565         boolean valid = importer.doImport(dc, req.getInputStream(), sheet, validateOnly, filename);
     568        boolean valid = importer.doImport(dc, req.getInputStream(), sheet, validateOnly, filename, progress);
     569        int numValid = importer.getNumValidEntries();
    566570        List<String> sheetNames = importer.getWorksheets();
    567571        if (sheetNames != null)
     
    570574          filename += "/" + importer.getParsedWorksheet();
    571575        }
     576       
     577        if (valid && !validateOnly)
     578        {
     579          int numCreated = importer.getNumCreated();
     580          int numExisting = importer.getNumExisting();
     581          if (numCreated > 0)
     582          {
     583            progress.display(95, "Saving " + numCreated + " new entries; " + numExisting + " already existed...");
     584          }
     585         
     586          dc.commit();
     587          progress.display(100, "Import complete. " + numValid + " entries processed; " + numCreated + " new; " + numExisting + " already existed.");
     588        }
    572589
    573590        if (valid)
    574591        {
    575           jsonMessages.add("Found " + importer.getNumEntries() + " entries in " + filename);
     592          jsonMessages.add("Found " + numValid + " entries in " + filename);
     593          if (!validateOnly)
     594          {
     595            jsonMessages.add(importer.getNumCreated() + " new entries created");
     596            jsonMessages.add(importer.getNumExisting() + " entries already existed");
     597          }
    576598        }
    577599        else
    578600        {
    579           jsonMessages.add("[Warning]"+importer.getNumValidEntries() + " of " + importer.getNumEntries() + " valid entries in " + filename);
     601          if (numValid > 0)
     602          {
     603            jsonMessages.add("[Warning]" + numValid + " of " + importer.getNumEntries() + " valid entries in " + filename);
     604          }
     605         
    580606          json.put("numErrors", importer.getErrorMessages().size());
    581607          jsonMessages.addAll(importer.getErrorMessages());
Note: See TracChangeset for help on using the changeset viewer.