Changeset 5911


Ignore:
Timestamp:
Apr 21, 2020, 9:37:05 AM (17 months ago)
Author:
Nicklas Nordborg
Message:

References #1234: Implement batch importer for followup blood samples

Added mappings for all columns and implemented several checks for sensible values.

Location:
extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/FollowupBloodImporter.java

    r5910 r5911  
    44import java.io.InputStream;
    55import java.util.ArrayList;
     6import java.util.Arrays;
     7import java.util.Date;
     8import java.util.HashMap;
     9import java.util.HashSet;
    610import java.util.List;
     11import java.util.Map;
     12import java.util.Set;
    713import java.util.regex.Pattern;
    814
    915import net.sf.basedb.core.DbControl;
     16import net.sf.basedb.util.Values;
    1017import net.sf.basedb.util.excel.XlsxToCsvUtil;
     18import net.sf.basedb.util.formatter.DateFormatter;
     19import net.sf.basedb.util.formatter.Formatter;
    1120import net.sf.basedb.util.parser.FlatFileParser;
    1221import net.sf.basedb.util.parser.FlatFileParser.LineType;
     
    2635  private String parsedWorksheet;
    2736
     37  private Set<String> allowedBloodSampleValues;
     38  private Pattern rccIdPattern;
     39  private Map<String, Integer> uniqueItems;
    2840  private int numEntries;
     41  private int numValid;
    2942 
    3043  public FollowupBloodImporter()
     
    3245    this.errorMessages = new ArrayList<String>();
    3346    this.warningMessages = new ArrayList<String>();
    34 
    3547  }
    3648 
     
    7890  {
    7991    return numEntries;
     92  }
     93 
     94  public int getNumValidEntries()
     95  {
     96    return numValid;
    8097  }
    8198 
     
    110127    String fileAndLine = "File '" + fileName + "' line " + lineNo + ": ";
    111128
    112     Mapper rccIdMapper = getRequiredMapper(ffp, "BloodRccidNumber", fileAndLine);
    113     Mapper pnrMapper = getRequiredMapper(ffp, "PersonalNumber", fileAndLine);
    114     Mapper caseMapper = getRequiredMapper(ffp, "Name", fileAndLine); // This is a Case name
     129    Formatter<Date> dateFormat = new DateFormatter("yyyy-MM-dd");
     130    Formatter<Date> timeFormat = new DateFormatter("hh:mm");
     131   
     132    Mapper rccIdMapper = getRequiredMapper(ffp, "BloodRccidNumber", null, fileAndLine);
     133    Mapper pnrMapper = getRequiredMapper(ffp, "PersonalNumber", null, fileAndLine);
     134    Mapper caseMapper = getRequiredMapper(ffp, "Name", null, fileAndLine); // This is a Case name
     135    Mapper bloodSampleMapper = getRequiredMapper(ffp, "BloodSample", null, fileAndLine);
     136    Mapper samplingDateMapper = getRequiredMapper(ffp, "BloodSamplingDate", dateFormat, fileAndLine);
     137    Mapper samplingTimeMapper = getRequiredMapper(ffp, "BloodSamplingTime", timeFormat, fileAndLine);
     138    Mapper freezerDateMapper = getRequiredMapper(ffp, "BloodFreezerDate", dateFormat, fileAndLine);
     139    Mapper freezerTimeMapper = getRequiredMapper(ffp, "BloodFreezerTime", timeFormat, fileAndLine);
     140    Mapper pathNoteMapper = getRequiredMapper(ffp, "OtherPathNote", timeFormat, fileAndLine);
     141   
    115142    // TODO -- more columns
    116143    if (hasError()) return false;
    117144
     145    // Keep track of RCC-ID values and check for duplicates
     146    uniqueItems = new HashMap<>();
     147    allowedBloodSampleValues = new HashSet<>(Arrays.asList("FollowUp06", "FollowUp12", "FollowUp36"));
     148    rccIdPattern = Pattern.compile("\\d{10}(B|C|D)"); // RCC-id should be 10 digits followed by B, C or D
     149   
    118150    while (ffp.hasMoreData())
    119151    {
     
    122154      fileAndLine = "File '" + fileName + "' line " + lineNo + ": ";
    123155      numEntries++;
     156     
     157      BloodEntry b = new BloodEntry(fileAndLine);
     158      b.lineNo = lineNo;
     159      b.rccId = rccIdMapper.getString(data);
     160      b.pnr = pnrMapper.getString(data);
     161      b.caseName = caseMapper.getString(data);
     162      b.bloodSample = bloodSampleMapper.getString(data);
     163      b.samplingDate = samplingDateMapper.getDate(data);
     164      b.samplingDateText = samplingDateMapper.getString(data);
     165      b.samplingTime = samplingTimeMapper.getDate(data);
     166      b.samplingTimeText = samplingTimeMapper.getString(data);
     167     
     168      b.freezerDate = freezerDateMapper.getDate(data);
     169      b.freezerDateText = freezerDateMapper.getString(data);
     170      b.freezerTime = freezerTimeMapper.getDate(data);
     171      b.freezerTimeText = freezerTimeMapper.getString(data);
     172      b.otherPathNote = pathNoteMapper.getString(data);
     173     
     174      boolean valid = checkBloodEntry(b);
     175      if (valid) numValid++;
     176
    124177    }
    125178   
     
    143196    an error message is logged and null is returned.
    144197  */
    145   private Mapper getRequiredMapper(FlatFileParser ffp, String col, String fileAndLine)
     198  private Mapper getRequiredMapper(FlatFileParser ffp, String col, Formatter<Date> dateFormat, String fileAndLine)
    146199  {
    147200    Mapper mapper = null;
     
    152205    else
    153206    {
    154       mapper = ffp.getMapper("\\" + col + "\\");
     207      mapper = ffp.getMapper("\\" + col + "\\", dateFormat, true);
    155208    }
    156209    return mapper;
     
    178231    return mapper;
    179232  }
    180 
     233 
     234  public boolean checkBloodEntry(BloodEntry b)
     235  {
     236    if (b.rccId == null)
     237    {
     238      addErrorMessage(b.fileAndLine+"Missing 'BloodRccidNumber'");
     239      return false;
     240    }
     241    else
     242    {
     243      Integer duplicateLine = uniqueItems.get(b.rccId);
     244      if (duplicateLine != null)
     245      {
     246        addErrorMessage(b.fileAndLine + "Duplicate 'BloodRccidNumber': " + b.rccId + " (also found on line " + duplicateLine + ")");
     247        return false;
     248      }
     249      uniqueItems.put(b.rccId, b.lineNo);
     250    }
     251    if (!rccIdPattern.matcher(b.rccId).matches())
     252    {
     253      addErrorMessage(b.fileAndLine + "Invalid 'BloodRccidNumber': " + b.rccId + " (expected format is 10 digits followed by B, C or D)");
     254      return false;
     255    }
     256
     257    if (b.pnr == null)
     258    {
     259      addErrorMessage(b.fileAndLine+"Missing 'PersonalNumber'");
     260      return false;
     261    }
     262    if (b.caseName == null)
     263    {
     264      addErrorMessage(b.fileAndLine+"Missing 'Name'");
     265      return false;
     266    }
     267   
     268    if (b.bloodSample == null)
     269    {
     270      addErrorMessage(b.fileAndLine+"Missing 'BloodSample'");
     271      return false;
     272    }
     273    else if (!allowedBloodSampleValues.contains(b.bloodSample))
     274    {
     275      addErrorMessage(b.fileAndLine+"Invalid 'BloodSample' value: " + b.bloodSample + " (expected one of '" + Values.getString(allowedBloodSampleValues, "', '", true) + "')");
     276      return false;
     277    }
     278   
     279    if (b.samplingDateText != null && b.samplingDate == null)
     280    {
     281      // The text could not be converted to a date
     282      addErrorMessage(b.fileAndLine+"Invalid 'BloodSamplingDate': " + b.samplingDateText + " (expected format is YYYY-MM-DD)");
     283      return false;
     284    }
     285    if (b.samplingTimeText != null && b.samplingTime == null)
     286    {
     287      addErrorMessage(b.fileAndLine+"Invalid 'BloodSamplingTime': " + b.samplingTimeText + " (expected format is HH:MM)");
     288      return false;
     289    }
     290
     291    if (b.freezerDateText != null && b.freezerDate == null)
     292    {
     293      addErrorMessage(b.fileAndLine+"Invalid 'BloodFreezerDate': " + b.freezerDateText + " (expected format is YYYY-MM-DD)");
     294      return false;
     295    }
     296    if (b.freezerTimeText != null && b.freezerTime == null)
     297    {
     298      addErrorMessage(b.fileAndLine+"Invalid 'BloodFreezerTime': " + b.freezerTimeText + " (expected format is HH:MM)");
     299      return false;
     300    }
     301   
     302    return true;
     303  }
     304
     305  public static class BloodEntry
     306  {
     307    final String fileAndLine;
     308    int lineNo;
     309    String pnr;
     310    String rccId;
     311    String caseName;
     312    String bloodSample;
     313    String samplingDateText;
     314    String samplingTimeText;
     315    Date samplingDate;
     316    Date samplingTime;
     317    String freezerDateText;
     318    String freezerTimeText;
     319    Date freezerDate;
     320    Date freezerTime;
     321    String otherPathNote;
     322   
     323    public BloodEntry(String fileAndLine)
     324    {
     325      this.fileAndLine = fileAndLine;
     326    }
     327  }
    181328 
    182329}
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/servlet/BloodFollowUpRegistrationServlet.java

    r5910 r5911  
    556556        String sheet = Values.getStringOrNull(req.getParameter("sheet"));
    557557       
     558        dc = sc.newDbControl();
    558559        if (!validateOnly)
    559560        {
    560           dc = sc.newDbControl();
    561561          ReggieRole.checkPermission(dc, "'" + cmd + "' wizard", ReggieRole.PATIENT_CURATOR, ReggieRole.ADMINISTRATOR);
    562562        }
     
    577577        else
    578578        {
     579          jsonMessages.add("[Warning]"+importer.getNumValidEntries() + " of " + importer.getNumEntries() + " valid entries in " + filename);
    579580          json.put("numErrors", importer.getErrorMessages().size());
    580581          jsonMessages.addAll(importer.getErrorMessages());
Note: See TracChangeset for help on using the changeset viewer.