Changeset 5117


Ignore:
Timestamp:
Nov 20, 2018, 4:09:52 PM (5 months ago)
Author:
Nicklas Nordborg
Message:

References #1081: The release exporter should export more data

The SpecimenWriter should now export the JPG image that is attached to the Stained item with GoodStain=true. It will write the image directly to the correct place on the release server. Except, if the same image already exists in an earlier release, it will instead create a symbolic link to it just as the exporter already do for other files.

Location:
extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release/BaseFileOutputLocation.java

    r4427 r5117  
    3636    Path p = new Path(rootPath + path, Path.Type.FILE);
    3737    Directory d = Directory.getNew(dc, p);
     38    d.setRemoved(false);
    3839   
    3940    File file = File.getFile(dc, d, p.getFilename(), true);
    4041    if (file.isInDatabase() && !getOverwrite() && !file.isRemoved())
    41     { 
     42    {
    4243      // Not allowed to overwrite the existing file
    4344      throw new ItemAlreadyExistsException("File[path="+file.getPath()+"]");     
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release/ReleaseExporter.java

    r5107 r5117  
    178178      jsonWriter.registerItemWriter(new CaseWriter(dc, options));
    179179      jsonWriter.registerItemWriter(new IncaWriter(dc, options));
    180       jsonWriter.registerItemWriter(new SpecimenWriter(dc, options));
     180      jsonWriter.registerItemWriter(new SpecimenWriter(dc, options, outputLocation, scriptWriter));
    181181      jsonWriter.registerItemWriter(new LysateWriter(dc, options));
    182182      jsonWriter.registerItemWriter(new RnaWriter(dc, options));
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release/ReleaseExporterPlugin.java

    r5106 r5117  
    129129        outputLocation = new BaseFileOutputLocation(dc, saveTo);
    130130        outputLocation.setOverwrite(Boolean.TRUE.equals(job.getValue("overwrite")));
    131         outputLocation.setCompress(true);
    132131      }
    133132     
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release/ScriptWriter.java

    r5106 r5117  
    131131  }
    132132 
     133  /**
     134    Add a file that is known to exists in an earlier release. The file
     135    will be softlinked in the mklinks.sh script.
     136    @param releasePath Path to folder on the release server
     137    @param filename The name of the file
     138    @param releasedVersion The version in which the relased file exists
     139    @since 4.21
     140  */
     141  public void addLinkToExistingFile(String releasePath, String filename, String releasedVersion)
     142  {
     143    addMkDir(releasePath);
     144    if (!releasePath.endsWith("/")) releasePath += "/";
     145    filesToLink.put(releasePath+filename, releasedVersion);
     146  }
     147 
    133148  public void writeScripts()
    134149  {
  • extensions/net.sf.basedb.reggie/trunk/src/net/sf/basedb/reggie/plugins/release/SpecimenWriter.java

    r5107 r5117  
    11package net.sf.basedb.reggie.plugins.release;
    22
     3import java.io.IOException;
     4import java.io.InputStream;
    35import java.time.LocalDateTime;
    46import java.time.temporal.ChronoUnit;
     
    911import org.json.simple.JSONObject;
    1012
     13import net.sf.basedb.core.AnyToAny;
    1114import net.sf.basedb.core.BioMaterialEvent;
    1215import net.sf.basedb.core.DbControl;
     16import net.sf.basedb.core.File;
    1317import net.sf.basedb.core.Item;
     18import net.sf.basedb.core.ItemNotFoundException;
    1419import net.sf.basedb.core.Quantity;
    1520import net.sf.basedb.core.Sample;
    1621import net.sf.basedb.core.Type;
    1722import net.sf.basedb.core.Unit;
     23import net.sf.basedb.core.plugin.ExportOutputStream;
    1824import net.sf.basedb.reggie.Site;
    1925import net.sf.basedb.reggie.converter.MultiplyFloatConverter;
     
    2228import net.sf.basedb.reggie.dao.SpecimenTube;
    2329import net.sf.basedb.reggie.json.FilteredJSONArray;
     30import net.sf.basedb.reggie.json.FilteredJSONObject;
     31import net.sf.basedb.util.FileUtil;
    2432import net.sf.basedb.util.filter.NotNullFilter;
    2533import net.sf.basedb.util.units.UnitUtil;
     
    3442
    3543  private final MultiplyFloatConverter toMilliG;
    36   public SpecimenWriter(DbControl dc, ReleaseWriterOptions options)
     44  private final OutputLocation location;
     45  private final ScriptWriter script;
     46 
     47  public SpecimenWriter(DbControl dc, ReleaseWriterOptions options, OutputLocation location, ScriptWriter script)
    3748  {
    3849    super(dc, options);
    3950    this.toMilliG = new MultiplyFloatConverter(0.001f); // Convert µg to mg
     51    this.location = location;
     52    this.script = script;
    4053  }
    4154 
     
    4760    List<SpecimenTube> specimens = item.getSpecimens();
    4861    if (specimens.size() == 0) return null;
     62    DbControl dc = item.getDbControl();
    4963   
    5064    JSONArray json = new JSONArray();
     
    108122      if (histology != null)
    109123      {
    110         jsonAnnotations.add(item.createAnnotationJSON("HisName", item.toReleaseId(histology.getName())));
    111       }
    112      
    113       if (goodStain != null)
    114       {
    115         jsonAnnotations.add(item.createAnnotationJSON("HisScoreComplete", item.getAnnotationValue(Annotationtype.SCORE_COMPLETE, goodStain)));
    116         jsonAnnotations.add(item.createAnnotationJSON("HisScoreInvasiveCancer", item.getAnnotationValue(Annotationtype.SCORE_INVASIVE_CANCER, goodStain)));
    117         jsonAnnotations.add(item.createAnnotationJSON("HisScoreInsituCancer", item.getAnnotationValue(Annotationtype.SCORE_INSITU_CANCER, goodStain)));
    118         jsonAnnotations.add(item.createAnnotationJSON("HisScoreLymphocytes", item.getAnnotationValue(Annotationtype.SCORE_LYMPHOCYTES, goodStain)));
    119         jsonAnnotations.add(item.createAnnotationJSON("HisScoreStroma", item.getAnnotationValue(Annotationtype.SCORE_STROMA, goodStain)));
    120         jsonAnnotations.add(item.createAnnotationJSON("HisScoreFat", item.getAnnotationValue(Annotationtype.SCORE_FAT, goodStain)));
    121         jsonAnnotations.add(item.createAnnotationJSON("HisScoreNormal", item.getAnnotationValue(Annotationtype.SCORE_NORMAL, goodStain)));
     124        String hisName = item.toReleaseId(histology.getName());
     125        jsonAnnotations.add(item.createAnnotationJSON("HisName", hisName));
     126       
     127        if (goodStain != null)
     128        {
     129          jsonAnnotations.add(item.createAnnotationJSON("HisScoreComplete", item.getAnnotationValue(Annotationtype.SCORE_COMPLETE, goodStain)));
     130          jsonAnnotations.add(item.createAnnotationJSON("HisScoreInvasiveCancer", item.getAnnotationValue(Annotationtype.SCORE_INVASIVE_CANCER, goodStain)));
     131          jsonAnnotations.add(item.createAnnotationJSON("HisScoreInsituCancer", item.getAnnotationValue(Annotationtype.SCORE_INSITU_CANCER, goodStain)));
     132          jsonAnnotations.add(item.createAnnotationJSON("HisScoreLymphocytes", item.getAnnotationValue(Annotationtype.SCORE_LYMPHOCYTES, goodStain)));
     133          jsonAnnotations.add(item.createAnnotationJSON("HisScoreStroma", item.getAnnotationValue(Annotationtype.SCORE_STROMA, goodStain)));
     134          jsonAnnotations.add(item.createAnnotationJSON("HisScoreFat", item.getAnnotationValue(Annotationtype.SCORE_FAT, goodStain)));
     135          jsonAnnotations.add(item.createAnnotationJSON("HisScoreNormal", item.getAnnotationValue(Annotationtype.SCORE_NORMAL, goodStain)));
     136         
     137          // Handle the image of the histology slide
     138          // On the Relax server they should be located in the top-level folder for the specimen
     139          // and have a name based on the HisName annotation
     140          String releaseDataFilesFolder = "/" + specimen.getExternalId();
     141          String imageName = hisName + ".jpg";
     142          String imagePath = releaseDataFilesFolder + "/" + imageName;
     143         
     144          // Check if the image already exists in the release archive
     145          String releasedInVersion = location.findReleasedFile(imagePath);
     146         
     147          JSONObject jsonImage = new FilteredJSONObject(new NotNullFilter<>(false));
     148          jsonImage.put("link", imageName);
     149          jsonImage.put("name", imageName);
     150          JSONArray jsonFiles = new JSONArray();
     151          if (releasedInVersion != null)
     152          {
     153            // We can safely let the script writer handle this since it should create a symbolic link
     154            jsonFiles.add(jsonImage);
     155            script.addLinkToExistingFile(releaseDataFilesFolder, imageName, releasedInVersion);
     156          }
     157          else
     158          {
     159            // We need to copy the images from the BASE file system to the release archive
     160            // BUT!!! we also get here if there is no image at all in which case we will get
     161            // exception and doesn't export any files at all
     162            InputStream imageIn = null;
     163            ExportOutputStream imageOut = null;
     164            try
     165            {
     166              // The image should be linked from the 'goodStain' sample via the name 'image'
     167              AnyToAny imageLink = AnyToAny.getByName(dc, goodStain, "image");
     168              File image = (File)imageLink.getTo();
     169              imageIn = image.getDownloadStream(0);
     170              imageOut = location.getOutputStream(imagePath, false);
     171              imageOut.setMimeType(image.getMimeType());
     172              FileUtil.copy(imageIn, imageOut);
     173              // We should copy the metadata as well
     174              jsonImage.put("size", image.getSize());
     175              jsonImage.put("mimeType", image.getMimeType());
     176              jsonImage.put("characterSet", image.getCharacterSet());
     177              jsonFiles.add(jsonImage);
     178            }
     179            catch (ItemNotFoundException | IOException ex)
     180            {} // Ignore since not all histology items have images yet
     181            finally
     182            {
     183              FileUtil.close(imageIn);
     184              FileUtil.close(imageOut);
     185            }
     186          }
     187          if (jsonFiles.size() > 0)
     188          {
     189            jsonAnnotations.add(item.createAnnotationJSON("DataFilesFolder", releaseDataFilesFolder));
     190            jsonSp.put("files", jsonFiles);
     191          }
     192        }
    122193      }
    123194     
     
    162233    spFactory.createAnnotationType(Annotationtype.LINKED_SPECIMEN).setProjectSpecificValues(true);
    163234
     235    // Need folder for Histology images
     236    spFactory.createAnnotationType(Annotationtype.DATA_FILES_FOLDER).setProjectSpecificValues(true);
     237   
    164238    spFactory.setNamePrefix("His");
    165239    spFactory.setProjectSpecificValues(true);
Note: See TracChangeset for help on using the changeset viewer.