Changeset 8094

Timestamp:

Nov 4, 2022, 9:05:30 AM (5 months ago)

Author:

Nicklas Nordborg

Message:

Merge BASE 3.19.5 to the trunk

Location:

trunk

Files:

• . (modified) (1 prop)
• .classpath (modified) (1 diff)
• doc/3rd-party-components.txt (modified) (2 diffs)
• doc/src/docbook/figures/configure_relateditems_columns.png (modified) (previous)
• doc/src/docbook/user/webclient.xml (modified) (1 diff)
• lib/dist/zip4j-2.11.2.jar (copied) (copied from tags/3.19.5/lib/dist/zip4j-2.11.2.jar)
• lib/dist/zip4j-license.txt (copied) (copied from tags/3.19.5/lib/dist/zip4j-license.txt)
• src/clients/web/net/sf/basedb/clients/web/extensions/list/RelatedItemColumn.java (modified) (7 diffs)
• src/clients/web/net/sf/basedb/clients/web/extensions/list/RelatedItemMultiHopColumn.java (modified) (2 diffs)
• src/core/net/sf/basedb/core/PropertyFilter.java (modified) (4 diffs)
• src/core/net/sf/basedb/util/listable/BioSourceToSampleTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/DerivedBioAssayToChildDerivedBioAssayTransformer.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/DerivedBioAssayToParentDerivedBioAssayTransformer.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/DerivedBioAssayToPhysicalBioAssayTransformer.java (modified) (3 diffs)
• src/core/net/sf/basedb/util/listable/DerivedBioAssayToRawBioAssayTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/ExtractToChildExtractTransformer.java (modified) (4 diffs)
• src/core/net/sf/basedb/util/listable/ExtractToParentExtractTransformer.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/ExtractToPhysicalBioAssayTransformer.java (modified) (3 diffs)
• src/core/net/sf/basedb/util/listable/ExtractToSampleTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/ListableUtil.java (modified) (3 diffs)
• src/core/net/sf/basedb/util/listable/PhysicalBioAssayToDerivedBioAssayTransformer.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/PhysicalBioAssayToExtractTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/RawBioAssayToDerivedBioAssayTransformer.java (modified) (3 diffs)
• src/core/net/sf/basedb/util/listable/SampleToBioSourceTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/SampleToChildSampleTransformer.java (modified) (5 diffs)
• src/core/net/sf/basedb/util/listable/SampleToExtractTransformer.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/listable/SampleToParentSampleTransformer.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/SourceItemTransformerChain.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/SourceItemTransformerWithCache.java (modified) (5 diffs)
• src/core/net/sf/basedb/util/listable/ToBioSourceSourceItemTransformerFactory.java (modified) (1 diff)
• src/core/net/sf/basedb/util/listable/ToExtractSourceItemTransformerFactory.java (modified) (4 diffs)
• src/core/net/sf/basedb/util/listable/ToSampleSourceItemTransformerFactory.java (modified) (3 diffs)
• src/core/net/sf/basedb/util/listable/TransformCache.java (modified) (6 diffs)
• src/core/net/sf/basedb/util/listable/TransformContext.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/zip/FilePacker.java (modified) (2 diffs)
• src/core/net/sf/basedb/util/zip/ZipFilePacker.java (modified) (5 diffs)
• src/plugins/core/net/sf/basedb/plugins/PackedFileExporter.java (modified) (3 diffs)
• www/common/columns/add_relateditem_column.js (modified) (5 diffs)
• www/common/columns/add_relateditem_column.jsp (modified) (3 diffs)
• www/images/child-item-doubleback.png (copied) (copied from tags/3.19.5/www/images/child-item-doubleback.png)
• www/include/styles/table.css (modified) (2 diffs)

trunk/.classpath

@@ -35 +35 @@
   <classpathentry kind="lib" path="lib/dist/poi-ooxml-5.0.0.jar"/>
   <classpathentry kind="lib" path="lib/dist/hibernate-core-5.5.6.Final.jar"/>
+  <classpathentry kind="lib" path="lib/dist/zip4j-2.11.2.jar"/>
   <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
     <attributes>

trunk/doc/3rd-party-components.txt

@@ -303 +303 @@
 Files    : AffxFusion.jar
 
+ZIP file support
+----------------
+A library for reading and writing ZIP files.
+
+More info: https://github.com/srikanth-lingala/zip4j
+Version  : 2.11.2
+License  : zip4j-license.txt
+Files    : zip4j-2.11.2.jar
+
+
 TAR file support
 ----------------
@@ -382 +392 @@
             commons-compress-1.20.jar, commons-math3-3.6.1.jar, xmlbeans-4.0.0.jar
 
+

trunk/doc/src/docbook/user/webclient.xml

@@ -2410 +2410 @@
           </listitem>
         </varlistentry>
+        
+        <varlistentry>
+          <term><guilabel>Allow doubling back</guilabel></term>
+          <listitem>
+            <para>        
+              The normal operation when both a parent and child selection 
+              has been made is that the path used when going up to the parent 
+              is ignored when going down to the child. If this option is checked
+              then it is allowed to "double back" the same path also when going
+              down to the child item.
+            </para>
+          </listitem>
+        </varlistentry>
 
         <varlistentry>

trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/RelatedItemColumn.java

@@ -146 +146 @@
     if (dc != null)
     {
-      SourceItemTransformerFactory transformerFactory = ListableUtil.getTransformerFactory(spec.targetType);
+      boolean includeChildrenThatPushToParents =  !spec.multiHop && 
+          (spec.getTargetType() == Item.EXTRACT || spec.getTargetType() == Item.SAMPLE) && 
+          spec.getItemSubtype(dc).getPushAnnotations();
+      SourceItemTransformerFactory transformerFactory = ListableUtil.getTransformerFactory(spec.targetType, includeChildrenThatPushToParents);
       SourceItemTransformer tmp = transformerFactory.create(spec.sourceType, spec.direction);
       Restriction targetRestriction = spec.createTargetTypeRestriction(dc);
@@ -186 +189 @@
   
   /**
+    Get the specification for this column.
+    @since 3.19.5
+  */
+  protected Specification getSpecification()
+  {
+    return spec;
+  }
+  
+  /**
     We finalize this implementation to make sure that the helper 
     implementation always get a chance to re-cycle transactions.
@@ -258 +270 @@
   {
     Set<Integer> sourceIds = Collections.singleton(item.getId());
+    helper.transformContext.resetCollected();
     if (preTransform != null)
     {
+      if (!spec.allowDoublingBack) helper.transformContext.setCollecting();
       sourceIds = preTransform.transform(helper.transformContext, sourceIds);
+      if (!spec.allowDoublingBack) helper.transformContext.setAvoiding();
     }
     Set<Integer> relatedIds = transformer.transform(helper.transformContext, sourceIds);
@@ -306 +321 @@
         baseIndex = 2;
         spec.directionRaw = tmp[1];
-        if ("CHILD".equals(spec.directionRaw)) spec.direction = SourceItemTransform.PARENT_TO_CHILD;
+        if (spec.directionRaw.endsWith("+")) spec.allowDoublingBack = true;
+        if (spec.directionRaw.startsWith("CHILD")) spec.direction = SourceItemTransform.PARENT_TO_CHILD;
       }
       spec.targetType = Item.valueOf(tmp[baseIndex]);
@@ -350 +366 @@
     String directionRaw;
     SourceItemTransform direction;
+    boolean allowDoublingBack;
     Item sourceType;
     Item targetType;
@@ -385 +402 @@
 
     /**
+      Enabled on a multi-hop path if the path down to the related child from the
+      parent item is allowed to traverse the same items as the path up from
+      the source child item to the parent.
+    */
+    public boolean isDoublingBackAllowed()
+    {
+      return allowDoublingBack;
+    }
+    
+    /**
       Get the source item type. This is not part of the expression, but
       is taken from the current list.
@@ -471 +498 @@
       result += direction == SourceItemTransform.CHILD_TO_PARENT ? "parentitem" : "childitem";
       if (multiHop) result += " multihop";
+      if (allowDoublingBack) result += " doublingback";
       result += "\">";
       result += StringUtil.coalesce(subtypeName, targetType.name());

trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/RelatedItemMultiHopColumn.java

@@ -18 +18 @@
 
   private final RelatedItemColumn nextHop;
+  private final Specification nextSpec;
+  
   RelatedItemMultiHopColumn(DbControl dc, int index, Specification spec, RelatedItemHelper helper, RelatedItemColumn nextHop)
   {
     super(dc, index, spec, helper);
     this.nextHop = nextHop;
+    this.nextSpec = nextHop.getSpecification();
     
     setValueType(nextHop.getValueType());
@@ -43 +46 @@
     setTitle(spec.generateTitle(nextHop.getTitle()));
     setSubtitle(nextHop.getSubtitle());
-    setTooltip(spec.generateTooltip(StringUtil.coalesce(nextHop.getTooltip(), nextHop.getTitle())));
+    setTooltip(spec.generateTooltip(StringUtil.coalesce(nextHop.getTooltip(), nextHop.getTitle())+
+      (nextSpec.allowDoublingBack ? " (doubling back allowed)" : " (no doubling back)")));
   }
 

trunk/src/core/net/sf/basedb/core/PropertyFilter.java

@@ -120 +120 @@
   private Unit unit;
   private boolean temporary;
+  private TransformContext transformContext;
   
   /**
@@ -866 +867 @@
       String[] parts = property.split("/", 5);
       SourceItemTransform direction = SourceItemTransform.PARENT_TO_CHILD;
+      boolean allowDoublingBack = false;
       int baseIndex = 1;
       if (parts.length == 5) // The /DIRECTION is optional
       {
         baseIndex = 2;
-        direction = "CHILD".equals(parts[1]) ? SourceItemTransform.CHILD_TO_PARENT : SourceItemTransform.PARENT_TO_CHILD;
+        if (parts[1].startsWith("CHILD")) direction = SourceItemTransform.CHILD_TO_PARENT;
+        allowDoublingBack = parts[1].endsWith("+");
       }
       Item targetType = Item.valueOf(parts[baseIndex]);
@@ -896 +899 @@
       }
       PropertyFilter pp = new PropertyFilter(targetProperty, operator, getValue(), getValueType());
+      // Create a TransformContext unless we already have one
+      // The subquery must use the same TransformContext in case we need the "doubling back" feature
+      TransformContext tCtx = transformContext == null ? new TransformContext(dc) : transformContext;
+      pp.transformContext = tCtx;
       ClientContext subContext = new ClientContext(dc);
       if (context != null) subContext.linkAttributes(context);
@@ -907 +914 @@
         cc.setException(targetItemContext.getException());
       }
-
+      
       // We then transform the list if ID:s to a list of child item ID:s using ItemList functionality
       SortedSet<Integer> parentIds = new TreeSet<>(subquery.idList(dc));
-      TransformContext tCtx = new TransformContext(dc);
 
       SourceItemTransformerFactory factory = ListableUtil.getTransformerFactory(query.getItemType());
-      factory = new SourceItemTransformerWithCache(factory);
+      factory = new SourceItemTransformerWithCache(factory, tCtx.isAvoiding() ? property+operator+value : null);
+      // If we are in a subquery the main query may need this for the "doubling back" feature
+      if (transformContext != null && !allowDoublingBack) transformContext.setCollecting();
       Set<Integer> myIds = factory.create(targetType, direction).transform(tCtx, parentIds);
+      // If we are in a subquery the main query may need this for the "doubling back" feature
+      if (transformContext != null && !allowDoublingBack) transformContext.setAvoiding();
       
       // The final ID list is returned as a restriction

trunk/src/core/net/sf/basedb/util/listable/BioSourceToSampleTransformer.java

@@ -56 +56 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Sample> query = Sample.getQuery();
     query.setIncludes(context.getInclude());
@@ -65 +66 @@
         Expressions.parameter("biosources")
       ));
-    return safeIdList(context.getDbControl(), query, "biosources", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "biosources", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/DerivedBioAssayToChildDerivedBioAssayTransformer.java

@@ -79 +79 @@
     while (parents.size() > 0)
     {
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

trunk/src/core/net/sf/basedb/util/listable/DerivedBioAssayToParentDerivedBioAssayTransformer.java

@@ -79 +79 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       
       // Store new parents and "used" children

trunk/src/core/net/sf/basedb/util/listable/DerivedBioAssayToPhysicalBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemQuery;
@@ -54 +53 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<PhysicalBioAssay> query = PhysicalBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +64 @@
     );
     
-    return safeIdList(context.getDbControl(), query, "bioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "bioAssays", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/DerivedBioAssayToRawBioAssayTransformer.java

@@ -67 +67 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     DbControl dc = context.getDbControl();
     
@@ -78 +79 @@
       )
     );
-    Set<Integer> all = safeIdList(dc, query, "bioAssays", source);
+    Set<Integer> all = context.avoid(getTargetItemType(), safeIdList(dc, query, "bioAssays", source));
     
     if (collectedExtracts != null)

trunk/src/core/net/sf/basedb/util/listable/ExtractToChildExtractTransformer.java

@@ -47 +47 @@
 
   private final boolean includeSourcesInTarget;
+  private final boolean pushOnly;
   
   /**
@@ -54 +55 @@
   public ExtractToChildExtractTransformer(boolean includeSourcesInTarget) 
   {
+    this(includeSourcesInTarget, false);
+  }
+  /**
+    Create a new extract to child extract transformer that only load children
+    has a subtype with "push annotations" set. 
+    @since 3.19.5
+  */
+  public ExtractToChildExtractTransformer(boolean includeSourcesInTarget, boolean childrensThatPushOnly)
+  {
     super(Item.EXTRACT, Item.EXTRACT);
     this.includeSourcesInTarget = includeSourcesInTarget;
+    this.pushOnly = childrensThatPushOnly;
   }
-  
+
   @Override
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
@@ -75 +86 @@
         Expressions.parameter("parents")
       ));
+    if (pushOnly)
+    {
+      query.join(Hql.innerJoin("itemSubtype", "st"));
+      query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
+    }
 
     // Keep track of all seen children and parents
@@ -85 +101 @@
     while (parents.size() > 0)
     {
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

trunk/src/core/net/sf/basedb/util/listable/ExtractToParentExtractTransformer.java

@@ -136 +136 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       if (upstream != null) parents.removeAll(upstream);
       if (downstream != null) parents.retainAll(downstream);

trunk/src/core/net/sf/basedb/util/listable/ExtractToPhysicalBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Item;
 import net.sf.basedb.core.ItemQuery;
@@ -53 +52 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<PhysicalBioAssay> query = PhysicalBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -65 +63 @@
         Expressions.parameter("parents")
       ));
-    return safeIdList(context.getDbControl(), query, "parents", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "parents", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/ExtractToSampleTransformer.java

@@ -74 +74 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Sample> query = Sample.getQuery();
     query.setIncludes(context.getInclude());
@@ -88 +89 @@
       query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
     }
-    return safeIdList(context.getDbControl(), query, "extracts", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "extracts", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/ListableUtil.java

@@ -27 +27 @@
 
 import net.sf.basedb.core.Item;
+import net.sf.basedb.core.ItemSubtype;
 import net.sf.basedb.core.Listable;
 import net.sf.basedb.core.SyncFilter.SourceItemTransform;
@@ -66 +67 @@
   public static SourceItemTransformerFactory getTransformerFactory(Item targetItemType)
   {
+    return getTransformerFactory(targetItemType, false);
+  }
+  
+  /**
+    Create a source item transformer factory that can transform items to the given
+    target item type. If no transformer factory exists, null is returned.
+    The 'includeChildrenThatPushToParent' parameter can be used for transforms
+    that go from child to parent items, when the target is a SAMPLE or EXTRACT.
+    When this flag is set, transformer will also include child items that have
+    a subtype that have the {@link ItemSubtype#getPushAnnotations()} flag enabled
+    in the final result.
+    
+    @since 3.19.5
+  */
+  public static SourceItemTransformerFactory getTransformerFactory(Item targetItemType, boolean includeChildrenThatPushToParent)
+  {
     SourceItemTransformerFactory factory = null;
     if (targetItemType == Item.BIOSOURCE)
@@ -73 +90 @@
     else if (targetItemType == Item.SAMPLE)
     {
-      factory = new ToSampleSourceItemTransformerFactory();
+      factory = new ToSampleSourceItemTransformerFactory(includeChildrenThatPushToParent);
     }
     else if (targetItemType == Item.EXTRACT)
     {
-      factory = new ToExtractSourceItemTransformerFactory();
+      factory = new ToExtractSourceItemTransformerFactory(includeChildrenThatPushToParent);
     }
     else if (targetItemType == Item.PHYSICALBIOASSAY)

trunk/src/core/net/sf/basedb/util/listable/PhysicalBioAssayToDerivedBioAssayTransformer.java

@@ -80 +80 @@
       )
     );
-    Set<Integer> all = safeIdList(dc, query, "bioAssays", source);
+    context.collect(getSourceItemType(), source);
+    Set<Integer> all = context.avoid(getTargetItemType(), safeIdList(dc, query, "bioAssays", source));
     
     if (collectedExtracts != null)

trunk/src/core/net/sf/basedb/util/listable/PhysicalBioAssayToExtractTransformer.java

@@ -55 +55 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Extract> query = Extract.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +67 @@
       )
     );
-    return safeIdList(context.getDbControl(), query, "bioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "bioAssays", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/RawBioAssayToDerivedBioAssayTransformer.java

@@ -24 +24 @@
 import java.util.Set;
 
-import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.DerivedBioAssay;
 import net.sf.basedb.core.Item;
@@ -55 +54 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
-    DbControl dc = context.getDbControl();
-    
+    context.collect(getSourceItemType(), source);
     ItemQuery<DerivedBioAssay> query = DerivedBioAssay.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +64 @@
       )
     );
-    return safeIdList(context.getDbControl(), query, "rawBioAssays", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "rawBioAssays", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/SampleToBioSourceTransformer.java

@@ -71 +71 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<BioSource> query = BioSource.getQuery();
     query.setIncludes(context.getInclude());
@@ -85 +86 @@
       query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
     }
-    return safeIdList(context.getDbControl(), query, "samples", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "samples", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/SampleToChildSampleTransformer.java

@@ -29 +29 @@
 import net.sf.basedb.core.ItemQuery;
 import net.sf.basedb.core.Sample;
-import net.sf.basedb.core.Type;
 import net.sf.basedb.core.query.Expressions;
 import net.sf.basedb.core.query.Hql;
@@ -48 +47 @@
 
   private final boolean includeSourcesInTarget;
+  private final boolean pushOnly;
   
   /**
@@ -55 +55 @@
   public SampleToChildSampleTransformer(boolean includeSourcesInTarget) 
   {
+    this(includeSourcesInTarget, false);
+  }
+  /**
+    Create a new sample to child sample transformer that only load children that
+    has a subtype with "push annotations" set. 
+    @since 3.19.5
+  */
+  public SampleToChildSampleTransformer(boolean includeSourcesInTarget, boolean childrensThatPushOnly)
+  {
     super(Item.SAMPLE, Item.SAMPLE);
     this.includeSourcesInTarget = includeSourcesInTarget;
+    this.pushOnly = childrensThatPushOnly;
   }
-  
+
   @Override
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
@@ -76 +86 @@
         Expressions.parameter("parents")
       ));
+    if (pushOnly)
+    {
+      query.join(Hql.innerJoin("itemSubtype", "st"));
+      query.restrict(Restrictions.eq(Hql.property("st", "pushAnnotations"), Expressions.bool(true)));
+    }
 
     // Keep track of all seen children and parents
@@ -86 +101 @@
     while (parents.size() > 0)
     {
-      query.setParameter("parents", parents, Type.INT);
-      Set<Integer> children = safeIdList(dc, query, "parents", parents);
+      context.collect(getSourceItemType(), parents);
+      Set<Integer> children = context.avoid(getTargetItemType(), safeIdList(dc, query, "parents", parents));
       
       // Store new children and "used" parents

trunk/src/core/net/sf/basedb/util/listable/SampleToExtractTransformer.java

@@ -57 +57 @@
   public Set<Integer> transform(TransformContext context, Set<Integer> source) 
   {
+    context.collect(getSourceItemType(), source);
     ItemQuery<Extract> query = Extract.getQuery();
     query.setIncludes(context.getInclude());
@@ -66 +67 @@
         Expressions.parameter("samples")
       ));
-    return safeIdList(context.getDbControl(), query, "samples", source);
+    return context.avoid(getTargetItemType(), safeIdList(context.getDbControl(), query, "samples", source));
   }
 

trunk/src/core/net/sf/basedb/util/listable/SampleToParentSampleTransformer.java

@@ -103 +103 @@
     while (children.size() > 0)
     {
-      Set<Integer> parents = safeIdList(dc, query, "children", children);
+      context.collect(getSourceItemType(), children);
+      Set<Integer> parents = context.avoid(getTargetItemType(), safeIdList(dc, query, "children", children));
       
       // Store new parents and "used" children

trunk/src/core/net/sf/basedb/util/listable/SourceItemTransformerChain.java

@@ -102 +102 @@
     for (SourceItemTransformer sit : chain)
     {
-      //System.out.println(sit + " before: " + sit.getSourceItemType() + "=" + source.size() +":" + source);
       source = sit.transform(context, source);
-      //System.out.println("After: " + sit.getTargetItemType() + "=" + source.size());
     }
     return source;

trunk/src/core/net/sf/basedb/util/listable/SourceItemTransformerWithCache.java

@@ -47 +47 @@
 
   private final SourceItemTransformerFactory factory;
+  private final String cacheRegion;
   
   /**
@@ -57 +58 @@
   public SourceItemTransformerWithCache(SourceItemTransformerFactory factory)
   {
-    this.factory = factory;
+    this(factory, null);
   }
 
+  public SourceItemTransformerWithCache(SourceItemTransformerFactory factory, String cacheRegion)
+  {
+    this.factory = factory;
+    this.cacheRegion = cacheRegion;
+  }
+  
+  
   @Override
   public Item getTargetItem() 
@@ -75 +83 @@
   public SourceItemTransformer create(Item sourceItemType, SourceItemTransform transform) 
   {
-    return new TransformerWithCache(factory.create(sourceItemType, transform), transform);
+    return new TransformerWithCache(factory.create(sourceItemType, transform), transform, cacheRegion);
   }
   
@@ -85 +93 @@
     private final SourceItemTransformer transformer;
     private final SourceItemTransform transform;
+    private final String cacheRegion;
     
-    TransformerWithCache(SourceItemTransformer transformer, SourceItemTransform transform)
+    TransformerWithCache(SourceItemTransformer transformer, SourceItemTransform transform, String cacheRegion)
     {
       this.transformer = transformer;
       this.transform = transform;
+      this.cacheRegion = cacheRegion;
     }
     
@@ -117 +127 @@
       if (cache != null)
       {
-        key = TransformCache.getKey(getSourceItemType(), getTargetItemType(), transform, source);
+        key = TransformCache.getKey(cacheRegion, getSourceItemType(), getTargetItemType(), transform, source);
         target = cache.get(key);
       }

trunk/src/core/net/sf/basedb/util/listable/ToBioSourceSourceItemTransformerFactory.java

@@ -68 +68 @@
 
       // Use utility function to add transformation up to the extract level
-      ToExtractSourceItemTransformerFactory.childToParentChain(chain, sourceItemType, Item.BIOSOURCE);
+      ToExtractSourceItemTransformerFactory.childToParentChain(chain, sourceItemType, Item.BIOSOURCE, false);
       if (chain.size() > 0) 
       {

trunk/src/core/net/sf/basedb/util/listable/ToExtractSourceItemTransformerFactory.java

@@ -50 +50 @@
     { Item.EXTRACT, Item.PHYSICALBIOASSAY, Item.DERIVEDBIOASSAY, Item.RAWBIOASSAY };
   
+  private final boolean includeChildrenThatPushToParent;
   
   public ToExtractSourceItemTransformerFactory() 
   {
+    this(false);
+  }
+  
+  public ToExtractSourceItemTransformerFactory(boolean includeChildrenThatPushToParent) 
+  {
     super(Item.EXTRACT, PARENT_TO_CHILD, CHILD_TO_PARENT);
+    this.includeChildrenThatPushToParent = includeChildrenThatPushToParent;
   }
   
@@ -93 +100 @@
     else if (transform == SourceItemTransform.CHILD_TO_PARENT)
     {
-      childToParentChain(chain, sourceItemType, Item.EXTRACT);
+      childToParentChain(chain, sourceItemType, Item.EXTRACT, includeChildrenThatPushToParent);
     }
 
@@ -105 +112 @@
     DERIVEDBIOASSAY, PHYSICALBIOASSAY and EXTRACT
   */
-  static void childToParentChain(List<SourceItemTransformer> chain, Item sourceItemType, Item targetItemType)
+  static void childToParentChain(List<SourceItemTransformer> chain, Item sourceItemType, Item targetItemType, boolean includeChildrenThatPushToParent)
   {
     Item stepBySource = null;
@@ -146 +153 @@
       // Load parent extracts, maybe including the source extracts
       chain.add(new ExtractToParentExtractTransformer(sourceItemType != targetItemType, collectedExtracts));
+      // Also include child extracts that have push-to-parent flag set
+      if (includeChildrenThatPushToParent && targetItemType == Item.EXTRACT)
+      {
+        chain.add(new ExtractToChildExtractTransformer(true, true));
+      }
     }
     

trunk/src/core/net/sf/basedb/util/listable/ToSampleSourceItemTransformerFactory.java

@@ -51 +51 @@
   
   
+  private final boolean includeChildrenThatPushToParent;
+  
   public ToSampleSourceItemTransformerFactory() 
   {
-    super(Item.SAMPLE, PARENT_TO_CHILD, CHILD_TO_PARENT);
+    this(false);
   }
   
-  
+  public ToSampleSourceItemTransformerFactory(boolean includeChildrenThatPushToParent) 
+  {
+    super(Item.SAMPLE, PARENT_TO_CHILD, CHILD_TO_PARENT);
+    this.includeChildrenThatPushToParent = includeChildrenThatPushToParent;
+  }
+
   @Override
   public SourceItemTransformer create(final Item sourceItemType, final SourceItemTransform transform) 
@@ -87 +94 @@
       
       // Use utility function to add transformation up to the extract level
-      ToExtractSourceItemTransformerFactory.childToParentChain(chain, sourceItemType, Item.SAMPLE);
+      ToExtractSourceItemTransformerFactory.childToParentChain(chain, sourceItemType, Item.SAMPLE, false);
       if (chain.size() > 0) 
       {
@@ -99 +106 @@
         // Load parent samples and maybe include source samples
         chain.add(new SampleToParentSampleTransformer(sourceItemType != Item.SAMPLE));
+        // Also include child samples that have push-to-parent flag set
+        if (includeChildrenThatPushToParent)
+        {
+          chain.add(new SampleToChildSampleTransformer(true, true));
+        }
       }
     }

trunk/src/core/net/sf/basedb/util/listable/TransformCache.java

@@ -30 +30 @@
   public static CacheKey getKey(Item sourceType, Item targetType, SyncFilter.SourceItemTransform transform, Set<Integer> sourceIds)
   {
+    return getKey(null, sourceType, targetType, transform, sourceIds);
+  }
+  
+  public static CacheKey getKey(String cacheRegion, Item sourceType, Item targetType, SyncFilter.SourceItemTransform transform, Set<Integer> sourceIds)
+  {
     SortedSet<Integer> sortedSrc = sourceIds instanceof SortedSet ? (SortedSet<Integer>)sourceIds : new TreeSet<>(sourceIds);
-    return new CacheKey(sourceType, targetType, transform, sortedSrc);
+    return new CacheKey(cacheRegion, sourceType, targetType, transform, sortedSrc);
   }
   
@@ -96 +101 @@
   public static class CacheKey
   {
+    private final String cacheRegion;
     private final Item sourceType;
     private final Item targetType;
@@ -101 +107 @@
     private final String sourceIdHash;
     
-    CacheKey(Item sourceType, Item targetType, SyncFilter.SourceItemTransform transform, SortedSet<Integer> sourceIds)
+    CacheKey(String cacheRegion, Item sourceType, Item targetType, SyncFilter.SourceItemTransform transform, SortedSet<Integer> sourceIds)
     {
+      this.cacheRegion = cacheRegion==null?"":cacheRegion;
       this.sourceType = sourceType;
       this.targetType = targetType;
@@ -112 +119 @@
     public int hashCode() 
     {
-      return sourceType.hashCode() + 3 * targetType.hashCode() + 7 * transform.hashCode() + 11 * sourceIdHash.hashCode();
+      return cacheRegion.hashCode()+sourceType.hashCode() + 3 * targetType.hashCode() + 7 * transform.hashCode() + 11 * sourceIdHash.hashCode();
     }
 
@@ -120 +127 @@
       if (!(obj instanceof CacheKey)) return false;
       CacheKey other = (CacheKey)obj;
-      return sourceType == other.sourceType && targetType == other.targetType && 
+      return cacheRegion.equals(other.cacheRegion) && 
+        sourceType == other.sourceType && targetType == other.targetType && 
         transform == other.transform && sourceIdHash.equals(other.sourceIdHash);
     }
@@ -127 +135 @@
     public String toString() 
     {
-      return "CacheKey[" + sourceType.name() + "->" + targetType.name() + "; " + transform.name() + "; " + sourceIdHash + "]";
-    }   
+      return "CacheKey["+cacheRegion + ";" + sourceType.name() + "->" + targetType.name() + "; " + transform.name() + "; " + sourceIdHash + "]";
+    }
   }
   

trunk/src/core/net/sf/basedb/util/listable/TransformContext.java

@@ -23 +23 @@
 
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import net.sf.basedb.core.DbControl;
 import net.sf.basedb.core.Include;
+import net.sf.basedb.core.Item;
 
 /**
@@ -89 +94 @@
   }
 
+  private boolean collecting;
+  private boolean avoiding;
+  private Map<Item, Set<Integer>> collectedByItemType;
+
+  
+  /**
+    Enable/disable collecting of all source items that the transformers in
+    this context are using.
+    @since 3.19.5
+  */
+  public void setCollecting(boolean collecting)
+  {
+    this.collecting = collecting;
+  }
+  
+  /**
+    Shortcut for enabling 'collecting' and disabling 'avoiding'.
+    @since 3.19.5
+  */
+  public void setCollecting()
+  {
+    this.collecting = true;
+    this.avoiding = false;
+  }
+  
+  public boolean isCollecting()
+  {
+    return collecting;
+  }
+  
+  /**
+    Enable/disable avoiding of returning any source items that has 
+    been previously collected.
+    @since 3.19.5
+  */
+  public void setAvoiding(boolean avoiding)
+  {
+    this.avoiding = avoiding;
+  }
+  /**
+    Shortcut for enabling 'avoiding' and disabling 'collecting'.
+    @since 3.19.5
+  */
+  public void setAvoiding()
+  {
+    this.collecting = false;
+    this.avoiding = true;
+  }
+
+  public boolean isAvoiding()
+  {
+    return avoiding;
+  }
+  
+  /**
+    Clear all collected items and disable both 'collecting' and
+    'avoiding'.
+    @since 3.19.5
+  */
+  public void resetCollected()
+  {
+    this.collecting = false;
+    this.avoiding = false;
+    if (collectedByItemType != null) collectedByItemType.clear();
+  }
+  
+  /**
+    Store collected items for later use if collecting has been enabled.
+    @return The source items
+  */
+  public Set<Integer> collect(Item itemType, Set<Integer> source)
+  {
+    if (collecting && source.size() > 0)
+    {
+      if (collectedByItemType == null) collectedByItemType = new HashMap<>();
+      Set<Integer> collected = collectedByItemType.get(itemType);
+      if (collected == null) 
+      {
+        collected = new HashSet<>();
+        collectedByItemType.put(itemType, collected);
+      }
+      collected.addAll(source);
+    }
+    return source;
+  }
+  
+  /**
+    Avoid collected items by removing them from the source set.
+    Note that this method modified the source set directly.
+    @return The source set
+  */
+  public Set<Integer> avoid(Item itemType, Set<Integer> source)
+  {
+    if (avoiding && source.size() > 0)
+    {
+      if (collectedByItemType != null && collectedByItemType.containsKey(itemType))
+      {
+        Set<Integer> collected = collectedByItemType.get(itemType);
+        source.removeAll(collected);
+      }
+    }
+    return source;
+  }
+
 }

trunk/src/core/net/sf/basedb/util/zip/FilePacker.java

@@ -27 +27 @@
 import java.io.InputStream;
 import java.io.OutputStream;
+
+import net.sf.basedb.core.PluginParameter;
+import net.sf.basedb.core.StringParameterType;
 
 /**
@@ -65 +68 @@
   
   /**
+    Does the packer support encryption or not?
+    The default implementation return false.
+    @since 3.19.5
+  */
+  public default boolean supportsEncryption()
+  {
+    return false;
+  }
+  
+  /**
+    The default implementation return a parameter asking for a password.
+    The implementing class may create another parameter as long as it is
+    named "password". This method is only called if the implementation
+    supports encryption.
+    @since 3.19.5
+  */
+  public default PluginParameter<String> getPasswordParameter()
+  {
+    PluginParameter<String> passwordParameter = new PluginParameter<String>
+    (
+      "password", "Password", "Enter a password to encrypt the archive." , new StringParameterType()
+    );
+    return passwordParameter;
+  }
+  
+  /**
+    Set the password to use for encrypting the archive. This method
+    is only called if the implementation supports encryption and the
+    user has entered a password.
+    @since 3.19.5
+  */
+  public default void setPassword(String password)
+  {}
+  
+  /**
     The output stream that the compressed files should be written to.
     @param out The output stream to write to

trunk/src/core/net/sf/basedb/util/zip/ZipFilePacker.java

@@ -23 +23 @@
 package net.sf.basedb.util.zip;
 
+import net.lingala.zip4j.io.outputstream.ZipOutputStream;
+import net.lingala.zip4j.model.ZipParameters;
+import net.lingala.zip4j.model.enums.AesKeyStrength;
+import net.lingala.zip4j.model.enums.CompressionLevel;
+import net.lingala.zip4j.model.enums.CompressionMethod;
+import net.lingala.zip4j.model.enums.EncryptionMethod;
+import net.sf.basedb.core.BaseException;
+import net.sf.basedb.core.PluginParameter;
+import net.sf.basedb.core.StringParameterType;
 import net.sf.basedb.util.FileUtil;
 
@@ -28 +37 @@
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 /**
@@ -42 +49 @@
 {
   private ZipOutputStream zip;
+  private String password;
   
   /**
@@ -78 +86 @@
   }
   /**
-   * Wrap the output stream in a {@link ZipOutputStream}.
-   */
+    Wrap the output stream in a {@link ZipOutputStream}.
+  */
   @Override
   public void setOutputStream(OutputStream out)
   {
-    this.zip = new ZipOutputStream(out);
-    zip.setMethod(ZipOutputStream.DEFLATED);
+    try
+    {
+      this.zip = password == null ? new ZipOutputStream(out) : new ZipOutputStream(out, password.toCharArray());
+    }
+    catch (IOException ex)
+    {
+      throw new BaseException(ex);
+    }
   }
+  
   /**
-    Create a new {@link ZipEntry} and write the compressed
-    data to it.
+    Encryption is supported.
+    @since 3.19.5
+  */
+  @Override
+  public boolean supportsEncryption() 
+  {
+    return true;
+  }
+
+  /**
+    Get the encyption password parameter.
+    @since 3.19.5
+  */
+  @Override
+  public PluginParameter<String> getPasswordParameter()
+  {
+    PluginParameter<String> passwordParameter = new PluginParameter<String>
+    (
+      "password", "Password", "Enter a password to encrypt the archive with strong AES256 encryption. "
+        + "If empty, the ZIP archive will not be encrypted." , new StringParameterType()
+    );
+    return passwordParameter;
+  }
+
+  /**
+    @since 3.19.5
+  */
+  @Override
+  public void setPassword(String password) 
+  {
+    this.password = password;
+  }
+
+  /**
+    Create a new entry and write the compressed data to it.
   */
   @Override
@@ -96 +144 @@
     boolean isDirectory = in == null;
     if (isDirectory && !entryName.endsWith("/")) entryName += "/";
-    ZipEntry entry = new ZipEntry(entryName);
-    if (lastModified > 0) entry.setTime(lastModified);
-    if (isDirectory) 
+    
+    ZipParameters zipParameters = new ZipParameters();
+    zipParameters.setFileNameInZip(entryName);
+    if (lastModified > 0)
     {
-      entry.setMethod(ZipOutputStream.STORED);
-      entry.setSize(0);
-      entry.setCrc(0);
+      zipParameters.setLastModifiedFileTime(lastModified);
+    }
+    if (isDirectory)
+    {
+      zipParameters.setCompressionMethod(CompressionMethod.STORE);
+      zipParameters.setEntrySize(0);
+      zipParameters.setEntryCRC(0);
     }
     else
     {
-      entry.setSize(size);
+      zipParameters.setCompressionMethod(CompressionMethod.DEFLATE);
+      zipParameters.setCompressionLevel(CompressionLevel.NORMAL);
+      zipParameters.setEntrySize(size);
+      if (password != null)
+      {
+        zipParameters.setEncryptFiles(true);
+        zipParameters.setEncryptionMethod(EncryptionMethod.AES);
+        zipParameters.setAesKeyStrength(AesKeyStrength.KEY_STRENGTH_256);
+      }
     }
-    zip.putNextEntry(entry);
+    zip.putNextEntry(zipParameters);
     if (!isDirectory) FileUtil.copy(in, zip);
     zip.flush();

trunk/src/plugins/core/net/sf/basedb/plugins/PackedFileExporter.java

@@ -308 +308 @@
         
         // Other options
+        if (packer.supportsEncryption())
+        {
+          storeValue(job, request, ri.getParameter("password"));
+        }
         storeValue(job, request, ri.getParameter("removeItems"));
         if (saveAs != null)
@@ -358 +362 @@
     List<Integer> directories = job.getValues("directories");
     boolean removeItems = Boolean.TRUE.equals(job.getValue("removeItems"));
+    String password = job.getValue("password");
     FilePacker packer = getPacker();
+    if (packer.supportsEncryption() && password != null)
+    {
+      packer.setPassword(password);
+    }
 
     ChainedProgressReporter chained = null;
@@ -528 +537 @@
         parameters.add(getSaveAsParameter(null, null, defaultPath, requireFile));
         parameters.add(getOverwriteParameter(null, null));
+        // Encryption
+        if (packer.supportsEncryption())
+        {
+          parameters.add(packer.getPasswordParameter());
+        }
 
         // Remove parameter

trunk/www/common/columns/add_relateditem_column.js

@@ -109 +109 @@
     configure.clearList('childItemType');
     configure.clearList('childSubtype');
+    Doc.element('allowDoublingBack').disabled = true;
     
     var frm = document.forms['relatedItems'];
@@ -200 +201 @@
     configure.clearColumns();
     configure.clearList('childSubtype');
+    Doc.element('allowDoublingBack').disabled = true;
     
     var frm = document.forms['relatedItems'];
@@ -267 +269 @@
       frm.childSubtype.selectedIndex = 0;
       frm.childSubtype.disabled = currentChildSubtypes.length == 0;
+      frm.allowDoublingBack.disabled = currentChildSubtypes.length == 0;
     } 
     
@@ -371 +374 @@
     data.childItemType = frm.childItemType.value;
     data.childSubtype = frm.childSubtype.value;
+    data.allowDoublingBack = frm.allowDoublingBack.checked?1:0;
 
     if (!data.targetItemType)
@@ -396 +400 @@
     {
       data.childSubtypeName = frm.childSubtype[frm.childSubtype.selectedIndex].text;
-      propertyPrefix += '/CHILD/'+data.childItemType+'/'+data.childSubtype+'/';
+      propertyPrefix += (data.allowDoublingBack?'/CHILD+/':'/CHILD/')+data.childItemType+'/'+data.childSubtype+'/';
       titlePrefix += '.'+data.childSubtypeName;
     }

trunk/www/common/columns/add_relateditem_column.jsp

@@ -58 +58 @@
 {
   width: 100%;
+  position: absolute;
+  border-collapse: separate;
+}
+
+.columnstable thead
+{
+  position: sticky;
+  top: 0;
 }
 
@@ -63 +71 @@
 {
   vertical-align: top;
+}
+
+.columnstable tr.bottomborder th
+{
+  border-bottom-width: 1px;
+}
+
+.columnstable tr.topborder th
+{
+  border-top-width: 1px;
 }
 
@@ -170 +188 @@
       </td>
     </tr>
+    <tr>
+      <th class="subprompt"></th>
+      <td>
+        <label><input type="checkbox" id="showAllAnnotationTypes" 
+          name="showAllAnnotationTypes">Show annotations for all subtypes</label>
+      </td>
+      <td></td>
+      <td>
+        <label title="Normally, child items that was passed when going up to the parent are ignored when going down. Enable this option to include all child items."
+          ><input type="checkbox" id="allowDoublingBack" name="allowDoublingBack" disabled
+          >Allow doubling back</label>
+      </td>
+    </tr>
     <tr class="dynamic">
       <th></th>
-      <td></td>
-      <td></td>
-      <td></td>
+      <td colspan="3" class="columnsFrom" style="padding-top: 2px;">
+        <span id="targetPath"></span><span id="childPath" class="childitem" style="display: none;"></span>
+      </td>
     </tr>
     </table>
     </div>
     
-    <div class="absolutefull" style="top: 4em; left: 10em;">
-      <div class="columnsFrom" style="height: 1.5em; white-space: nowrap; overflow: hidden;">
-        <span id="targetPath"></span><span id="childPath" class="childitem" style="display: none;"></span>
-        <label style="font-weight: normal;"><input type="checkbox" id="showAllAnnotationTypes" 
-          name="showAllAnnotationTypes">Show annotations for all subtypes</label>
-      </div>
+    <div class="absolutefull" style="top: 6.75em; left: 10em;">   
       <table id="columnstable" class="columnstable" style="width: 100%;">
-      <thead class="bg-filled-100 topborder bottomborder">
-        <tr>
+      <thead class="">
+        <tr class="bg-filled-100 topborder bottomborder">
           <th style="width: 33%;">Columns</th>
           <th style="width: 33%;">Annotations</th>

trunk/www/include/styles/table.css

@@ -253 +253 @@
 }
 
-/* A column header defining a parent/child item value */
-.itemlist div.data th.relateditemcol::before
-{
-  margin-right: 2px;
-  float: left;
-}
-
 .itemlist div.data th.relateditemcol span.parentitem::before
 {
@@ -274 +267 @@
 }
 
-
-/* This will make room for the '›' without causing the header to wrap */
-.itemlist div.data th.relateditemcol > span
-{
-  display: inline-block;
-  max-width: calc(100% - 16px);
-  overflow: hidden;
-}
+.itemlist div.data th.relateditemcol span.childitem.doublingback::before
+{
+  content: url('../../images/child-item-doubleback.png');
+}
+
 
 /* A column header defining a linked item value */