Changeset 8087 for branches/3.19-stable

Timestamp:

Oct 31, 2022, 9:25:23 AM (7 months ago)

Author:

Nicklas Nordborg

Message:

References #2286: Improve performance for multi-hop child->parent->child kolumns

Implemented support for keeping track of all items that are handled by a SourceItemTransformer chain. The information is stored in the TransformContext and can be used to "avoid" items when doing a second transform using the same context.

So, when doing a child->parent->child transform, during the first phase from child->parent the context will remember all items if the TransformContext.setCollecting() method has been called. During the second phase from parent->child doubling back can be avoided by calling TransformContext.setAvoiding().

Location:

branches/3.19-stable/src/core/net/sf/basedb/util/listable

Files:

• BioSourceToSampleTransformer.java (modified) (2 diffs)
• DerivedBioAssayToChildDerivedBioAssayTransformer.java (modified) (1 diff)
• DerivedBioAssayToParentDerivedBioAssayTransformer.java (modified) (1 diff)
• DerivedBioAssayToPhysicalBioAssayTransformer.java (modified) (3 diffs)
• DerivedBioAssayToRawBioAssayTransformer.java (modified) (2 diffs)
• ExtractToChildExtractTransformer.java (modified) (1 diff)
• ExtractToParentExtractTransformer.java (modified) (1 diff)
• ExtractToPhysicalBioAssayTransformer.java (modified) (3 diffs)
• ExtractToSampleTransformer.java (modified) (2 diffs)
• PhysicalBioAssayToDerivedBioAssayTransformer.java (modified) (1 diff)
• PhysicalBioAssayToExtractTransformer.java (modified) (2 diffs)
• RawBioAssayToDerivedBioAssayTransformer.java (modified) (3 diffs)
• SampleToBioSourceTransformer.java (modified) (2 diffs)
• SampleToChildSampleTransformer.java (modified) (2 diffs)
• SampleToExtractTransformer.java (modified) (2 diffs)
• SampleToParentSampleTransformer.java (modified) (1 diff)
• SourceItemTransformerChain.java (modified) (1 diff)
• TransformContext.java (modified) (2 diffs)

branches/3.19-stable/src/core/net/sf/basedb/util/listable/BioSourceToSampleTransformer.java

@@ -56 +56 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Sample> query = Sample.getQuery();
     query.setIncludes(context.getInclude());
@@ -65 +66 @@
         Expressions.parameter("biosources")
       ));
-    return safeIdList(context.getDbControl(), query, "biosources", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "biosources", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/DerivedBioAssayToChildDerivedBioAssayTransformer.java

@@ -79 +79 @@
     while (parents.size() > 0)
     {
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

branches/3.19-stable/src/core/net/sf/basedb/util/listable/DerivedBioAssayToParentDerivedBioAssayTransformer.java

@@ -79 +79 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       
       // Store new parents and "used" children

branches/3.19-stable/src/core/net/sf/basedb/util/listable/DerivedBioAssayToPhysicalBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemQuery;
@@ -54 +53 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<PhysicalBioAssay> query = PhysicalBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +64 @@
     );
     
-    return safeIdList(context.getDbControl(), query, "bioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "bioAssays", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/DerivedBioAssayToRawBioAssayTransformer.java

@@ -67 +67 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     DbControl dc = context.getDbControl();
     
@@ -78 +79 @@
       )
     );
-    Set<Integer> all = safeIdList(dc, query, "bioAssays", source);
+    Set<Integer> all = context.avoid(getTargetItemType(), safeIdList(dc, query, "bioAssays", source));
     
     if (collectedExtracts != null)

branches/3.19-stable/src/core/net/sf/basedb/util/listable/ExtractToChildExtractTransformer.java

@@ -101 +101 @@
     while (parents.size() > 0)
     {
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

branches/3.19-stable/src/core/net/sf/basedb/util/listable/ExtractToParentExtractTransformer.java

@@ -136 +136 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       if (upstream != null) parents.removeAll(upstream);
       if (downstream != null) parents.retainAll(downstream);

branches/3.19-stable/src/core/net/sf/basedb/util/listable/ExtractToPhysicalBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemQuery;
@@ -53 +52 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<PhysicalBioAssay> query = PhysicalBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -65 +63 @@
         Expressions.parameter("parents")
       ));
-    return safeIdList(context.getDbControl(), query, "parents", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "parents", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/ExtractToSampleTransformer.java

@@ -74 +74 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Sample> query = Sample.getQuery();
     query.setIncludes(context.getInclude());
@@ -88 +89 @@
       query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
     }
-    return safeIdList(context.getDbControl(), query, "extracts", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "extracts", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/PhysicalBioAssayToDerivedBioAssayTransformer.java

@@ -80 +80 @@
       )
     );
-    Set<Integer> all = safeIdList(dc, query, "bioAssays", source);
+    context.collect(getSourceItemType(), source);
+    Set<Integer> all = context.avoid(getTargetItemType(), safeIdList(dc, query, "bioAssays", source));
     
     if (collectedExtracts != null)

branches/3.19-stable/src/core/net/sf/basedb/util/listable/PhysicalBioAssayToExtractTransformer.java

@@ -55 +55 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Extract> query = Extract.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +67 @@
       )
     );
-    return safeIdList(context.getDbControl(), query, "bioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "bioAssays", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/RawBioAssayToDerivedBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.DerivedBioAssay;
 import net.sf.basedb.core.Item;
@@ -55 +54 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<DerivedBioAssay> query = DerivedBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +64 @@
       )
     );
-    return safeIdList(context.getDbControl(), query, "rawBioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "rawBioAssays", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/SampleToBioSourceTransformer.java

@@ -71 +71 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<BioSource> query = BioSource.getQuery();
     query.setIncludes(context.getInclude());
@@ -85 +86 @@
       query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
     }
-    return safeIdList(context.getDbControl(), query, "samples", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "samples", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/SampleToChildSampleTransformer.java

@@ -29 +29 @@
 import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Sample;
-import net.sf.basedb.core.Type;
 import net.sf.basedb.core.query.Expressions;
 import net.sf.basedb.core.query.Hql;
@@ -102 +101 @@
     while (parents.size() > 0)
     {
-      query.setParameter("parents", parents, Type.INT);
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

branches/3.19-stable/src/core/net/sf/basedb/util/listable/SampleToExtractTransformer.java

@@ -57 +57 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Extract> query = Extract.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +67 @@
         Expressions.parameter("samples")
       ));
-    return safeIdList(context.getDbControl(), query, "samples", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "samples", source));
   }
 

branches/3.19-stable/src/core/net/sf/basedb/util/listable/SampleToParentSampleTransformer.java

@@ -103 +103 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       
       // Store new parents and "used" children

branches/3.19-stable/src/core/net/sf/basedb/util/listable/SourceItemTransformerChain.java

@@ -102 +102 @@
     for (SourceItemTransformer sit : chain)
     {
-      //System.out.println(sit + " before: " + sit.getSourceItemType() + "=" + source.size() +":" + source);
       source = sit.transform(context, source);
-      //System.out.println("After: " + sit.getTargetItemType() + "=" + source.size());
     }
     return source;

branches/3.19-stable/src/core/net/sf/basedb/util/listable/TransformContext.java

@@ -23 +23 @@
 
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Include;
+import net.sf.basedb.core.Item;
 
 /**
@@ -89 +94 @@
   }
 
+  private boolean collecting;
+  private boolean avoiding;
+  private Map<Item, Set<Integer>> collectedByItemType;
+
+  
+  /**
+    Enable/disable collecting of all source items that the transformers in
+    this context are using.
+    @since 3.19.5
+  */
+  public void setCollecting(boolean collecting)
+  {
+    this.collecting = collecting;
+  }
+  
+  /**
+    Shortcut for enabling 'collecting' and disabling 'avoiding'.
+    @since 3.19.5
+  */
+  public void setCollecting()
+  {
+    this.collecting = true;
+    this.avoiding = false;
+  }
+  
+  public boolean isCollecting()
+  {
+    return collecting;
+  }
+  
+  /**
+    Enable/disable avoiding of returning any source items that has 
+    been previously collected.
+    @since 3.19.5
+  */
+  public void setAvoiding(boolean avoiding)
+  {
+    this.avoiding = avoiding;
+  }
+  /**
+    Shortcut for enabling 'avoiding' and disabling 'collecting'.
+    @since 3.19.5
+  */
+  public void setAvoiding()
+  {
+    this.collecting = false;
+    this.avoiding = true;
+  }
+
+  public boolean isAvoiding()
+  {
+    return avoiding;
+  }
+  
+  /**
+    Clear all collected items and disable both 'collecting' and
+    'avoiding'.
+    @since 3.19.5
+  */
+  public void resetCollected()
+  {
+    this.collecting = false;
+    this.avoiding = false;
+    if (collectedByItemType != null) collectedByItemType.clear();
+  }
+  
+  /**
+    Store collected items for later use if collecting has been enabled.
+    @return The source items
+  */
+  public Set<Integer> collect(Item itemType, Set<Integer> source)
+  {
+    if (collecting && source.size() > 0)
+    {
+      if (collectedByItemType == null) collectedByItemType = new HashMap<>();
+      Set<Integer> collected = collectedByItemType.get(itemType);
+      if (collected == null) 
+      {
+        collected = new HashSet<>();
+        collectedByItemType.put(itemType, collected);
+      }
+      collected.addAll(source);
+    }
+    return source;
+  }
+  
+  /**
+    Avoid collected items by removing them from the source set.
+    Note that this method modified the source set directly.
+    @return The source set
+  */
+  public Set<Integer> avoid(Item itemType, Set<Integer> source)
+  {
+    if (avoiding && source.size() > 0)
+    {
+      if (collectedByItemType != null && collectedByItemType.containsKey(itemType))
+      {
+        Set<Integer> collected = collectedByItemType.get(itemType);
+        source.removeAll(collected);
+      }
+    }
+    return source;
+  }
+
 }