Changeset 6586

Timestamp:

Oct 31, 2014, 11:13:09 AM (9 years ago)

Author:

Nicklas Nordborg

Message:

Fixes #1878: Reduce memory footprint in raw data importer

FeatureInfo instances are used to store the information we need after the preload. Single instances of ReporterProxy and FeatureProxy are then populated with required values as they are needed.

Note that this only works because the batcher will process each raw data entry immediately and add it to the batch queue. If it had not, then the next raw data entry would have overwritten the values for the one before it causing incorrect reporter/feature mappings for the raw data.

File:

• branches/3.3-stable/src/core/net/sf/basedb/core/RawDataBatcher.java (modified) (8 diffs)

branches/3.3-stable/src/core/net/sf/basedb/core/RawDataBatcher.java

@@ -39 +39 @@
 import java.util.Set;
 import java.sql.SQLException;
+
 
 /**
@@ -161 +162 @@
     @see FeatureIdentificationMethod
   */
-  private Map<Object, FeatureData> preloaded;
+  private Map<Object, FeatureInfo> preloaded;
+  
+  private ReporterProxy reporterProxy;
+  private FeatureProxy featureProxy;
   
   /**
@@ -523 +527 @@
           "] has already been used by another spot.");
       }
-      FeatureData feature = preloaded.get(featureId);
+      FeatureInfo feature = preloaded.get(featureId);
       if (feature == null)
       {
@@ -531 +535 @@
             "=" + featureId + "] doesn't exist on array design");
       }
-      ReporterData reporterOnFeature = feature.getReporter();
+      ReporterData reporterOnFeature = feature.getReporter(reporterProxy);
       if (reporterOnFeature == null && acceptInsertIfNullReporterOnFeature)
       {
@@ -547 +551 @@
       }
       data.setPosition(feature.getPosition());
-      setPropertyValue(data, "feature", feature);
+      setPropertyValue(data, "feature", feature.getFeature(featureProxy));
       data.setReporter(reporterOnFeature);
     }
@@ -567 +571 @@
       query.setInteger("arrayDesign", arrayDesign.getId());
       int numFeatures = HibernateUtil.loadData(Long.class, query).intValue();
-      preloaded = new HashMap<Object, FeatureData>(numFeatures);
+      preloaded = new HashMap<Object, FeatureInfo>(numFeatures);
+      reporterProxy = new ReporterProxy();
+      featureProxy = new FeatureProxy();
       query = HibernateUtil.getPredefinedQuery(dc.getStatelessSession(), "PRELOAD_FEATURES");
       /*
@@ -592 +598 @@
         FeatureData feature = si.next();
         Object featureId = fiMethod.getIdentifier(feature);
-        if (preloaded.put(featureId, feature) != null)
+        if (preloaded.put(featureId, new FeatureInfo(feature)) != null)
         {
           throw new InvalidDataException("Can't identify features with " + fiMethod + 
@@ -652 +658 @@
   }
 
+
+  static class FeatureInfo
+  {
+    private final int featureId;
+    private final int position;
+    private final int reporterId;
+    private final String externalReporterId;
+
+    FeatureInfo(FeatureData feature)
+    {
+      featureId = feature.getId();
+      position = feature.getPosition();
+      ReporterData reporter = feature.getReporter();
+      if (reporter != null)
+      {
+        reporterId = reporter.getId();
+        externalReporterId = reporter.getExternalId();
+      }
+      else
+      {
+        reporterId = 0;
+        externalReporterId = null;
+      }
+    }
+    
+    int getPosition()
+    {
+      return position;
+    }
+    
+    ReporterData getReporter(ReporterProxy proxy)
+    {
+      return reporterId == 0 ? null : proxy.setTheId(reporterId, externalReporterId);
+    }
+    
+    FeatureData getFeature(FeatureProxy proxy)
+    {
+      return proxy.setTheId(featureId);
+    }
+    
+  }
+
+  static class ReporterProxy
+    extends ReporterData
+  {
+    private int id;
+    private String externalId;
+    
+    ReporterProxy()
+    {}
+    
+    ReporterProxy setTheId(int id, String externalId)
+    {
+      this.id = id;
+      this.externalId = externalId;
+      return this;
+    }
+    @Override
+    public int getId()
+    {
+      return id;
+    }
+    
+    @Override
+    public String getExternalId()
+    {
+      return externalId;
+    }
+  }
+
+  static class FeatureProxy
+    extends FeatureData
+  {
+    private int id;
+    
+    FeatureProxy()
+    {
+      super(null, (ReporterData)null);
+    }
+    
+    FeatureProxy setTheId(int id)
+    {
+      this.id = id;
+      return this;
+    }
+    @Override
+    public int getId()
+    {
+      return id;
+    }
+  }
+  
 }