Changeset 6586


Ignore:
Timestamp:
Oct 31, 2014, 11:13:09 AM (7 years ago)
Author:
Nicklas Nordborg
Message:

Fixes #1878: Reduce memory footprint in raw data importer

FeatureInfo instances are used to store the information we need after the preload. Single instances of ReporterProxy and FeatureProxy are then populated with required values as they are needed.

Note that this only works because the batcher will process each raw data entry immediately and add it to the batch queue. If it had not, then the next raw data entry would have overwritten the values for the one before it causing incorrect reporter/feature mappings for the raw data.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/3.3-stable/src/core/net/sf/basedb/core/RawDataBatcher.java

    r6582 r6586  
    3939import java.util.Set;
    4040import java.sql.SQLException;
     41
    4142
    4243/**
     
    161162    @see FeatureIdentificationMethod
    162163  */
    163   private Map<Object, FeatureData> preloaded;
     164  private Map<Object, FeatureInfo> preloaded;
     165 
     166  private ReporterProxy reporterProxy;
     167  private FeatureProxy featureProxy;
    164168 
    165169  /**
     
    523527          "] has already been used by another spot.");
    524528      }
    525       FeatureData feature = preloaded.get(featureId);
     529      FeatureInfo feature = preloaded.get(featureId);
    526530      if (feature == null)
    527531      {
     
    531535            "=" + featureId + "] doesn't exist on array design");
    532536      }
    533       ReporterData reporterOnFeature = feature.getReporter();
     537      ReporterData reporterOnFeature = feature.getReporter(reporterProxy);
    534538      if (reporterOnFeature == null && acceptInsertIfNullReporterOnFeature)
    535539      {
     
    547551      }
    548552      data.setPosition(feature.getPosition());
    549       setPropertyValue(data, "feature", feature);
     553      setPropertyValue(data, "feature", feature.getFeature(featureProxy));
    550554      data.setReporter(reporterOnFeature);
    551555    }
     
    567571      query.setInteger("arrayDesign", arrayDesign.getId());
    568572      int numFeatures = HibernateUtil.loadData(Long.class, query).intValue();
    569       preloaded = new HashMap<Object, FeatureData>(numFeatures);
     573      preloaded = new HashMap<Object, FeatureInfo>(numFeatures);
     574      reporterProxy = new ReporterProxy();
     575      featureProxy = new FeatureProxy();
    570576      query = HibernateUtil.getPredefinedQuery(dc.getStatelessSession(), "PRELOAD_FEATURES");
    571577      /*
     
    592598        FeatureData feature = si.next();
    593599        Object featureId = fiMethod.getIdentifier(feature);
    594         if (preloaded.put(featureId, feature) != null)
     600        if (preloaded.put(featureId, new FeatureInfo(feature)) != null)
    595601        {
    596602          throw new InvalidDataException("Can't identify features with " + fiMethod +
     
    652658  }
    653659
     660
     661  static class FeatureInfo
     662  {
     663    private final int featureId;
     664    private final int position;
     665    private final int reporterId;
     666    private final String externalReporterId;
     667
     668    FeatureInfo(FeatureData feature)
     669    {
     670      featureId = feature.getId();
     671      position = feature.getPosition();
     672      ReporterData reporter = feature.getReporter();
     673      if (reporter != null)
     674      {
     675        reporterId = reporter.getId();
     676        externalReporterId = reporter.getExternalId();
     677      }
     678      else
     679      {
     680        reporterId = 0;
     681        externalReporterId = null;
     682      }
     683    }
     684   
     685    int getPosition()
     686    {
     687      return position;
     688    }
     689   
     690    ReporterData getReporter(ReporterProxy proxy)
     691    {
     692      return reporterId == 0 ? null : proxy.setTheId(reporterId, externalReporterId);
     693    }
     694   
     695    FeatureData getFeature(FeatureProxy proxy)
     696    {
     697      return proxy.setTheId(featureId);
     698    }
     699   
     700  }
     701
     702  static class ReporterProxy
     703    extends ReporterData
     704  {
     705    private int id;
     706    private String externalId;
     707   
     708    ReporterProxy()
     709    {}
     710   
     711    ReporterProxy setTheId(int id, String externalId)
     712    {
     713      this.id = id;
     714      this.externalId = externalId;
     715      return this;
     716    }
     717    @Override
     718    public int getId()
     719    {
     720      return id;
     721    }
     722   
     723    @Override
     724    public String getExternalId()
     725    {
     726      return externalId;
     727    }
     728  }
     729
     730  static class FeatureProxy
     731    extends FeatureData
     732  {
     733    private int id;
     734   
     735    FeatureProxy()
     736    {
     737      super(null, (ReporterData)null);
     738    }
     739   
     740    FeatureProxy setTheId(int id)
     741    {
     742      this.id = id;
     743      return this;
     744    }
     745    @Override
     746    public int getId()
     747    {
     748      return id;
     749    }
     750  }
     751 
    654752}
Note: See TracChangeset for help on using the changeset viewer.