Changeset 7767


Ignore:
Timestamp:
Feb 6, 2020, 1:33:37 PM (22 months ago)
Author:
Nicklas Nordborg
Message:

References #2202: Include information from parent items in list pages

Reducing memory usage when exporting large data sets by using separate read-only transactions for loading parent items that can ne re-cycled at regular intervals.

Location:
trunk/src
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/ParentItemAnnotationColumn.java

    r7766 r7767  
    66import net.sf.basedb.core.Annotatable;
    77import net.sf.basedb.core.AnnotationType;
    8 import net.sf.basedb.core.DbControl;
    98import net.sf.basedb.core.snapshot.AnnotationSnapshot;
    109import net.sf.basedb.core.snapshot.AnnotationTypeFilter;
    … …  
    3130 
    3231  @Override
    33   public Object getValue(DbControl dc, Annotatable item)
     32  public Object getValue(ParentItemHelper helper, Annotatable item)
    3433  {
    35     Set<Annotatable> parents = getParentItems(dc, item);
     34    Set<Annotatable> parents = getParentItems(item);
    3635    if (parents.size() == 0) return null;
    3736   
    … …  
    3938    for (Annotatable p : parents)
    4039    {
    41       for (AnnotationSnapshot ss : helper.manager.findAnnotations(dc, p, atFilter, true, false))
     40      for (AnnotationSnapshot ss : helper.manager.findAnnotations(helper.dc, p, atFilter, true, false))
    4241      {
    4342        values.addAll(ss.getActualValues());
    … …  
    4746  }
    4847
    49 
    5048}

  • trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/ParentItemColumn.java

    r7766 r7767  
    1111import net.sf.basedb.core.ItemSubtype;
    1212import net.sf.basedb.core.Metadata;
    13 import net.sf.basedb.core.StringUtil;
    1413import net.sf.basedb.core.Metadata.PropertyPath;
    1514import net.sf.basedb.util.AnnotationUtil;
    … …  
    3837    /ITEMTYPE/subtype-id/property
    3938  */
     39  @SuppressWarnings({ "unchecked", "rawtypes" })
    4040  public static ParentItemColumn create(DbControl dc, String expr, ParentItemHelper helper)
    4141  {
    42     String[] tmp = expr.split("/");
    43     Item parentType = Item.valueOf(tmp[1]);
    44     ItemSubtype subtype = ItemSubtype.getById(dc, Values.getInt(tmp[2]));
    45     Filter<Annotatable> parentFilter = new SubtypeFilter(subtype);
    4642    ParentItemColumn col = null;
    47     if (tmp[3].startsWith("#"))
     43    try
    4844    {
    49       AnnotationType at = AnnotationType.getById(dc, Values.getInt(tmp[3].substring(1)));
    50       col = new ParentItemAnnotationColumn(expr, parentFilter, helper, at);
    51       col.setTitle(at.getName() + " [A]");
    52       col.setTooltip(subtype.getName() + " › "+at.getName());
    53       col.setValueType(at.getValueType());
    54       col.setExportFormatter(FormatterFactory.getTypeFormatter(dc.getSessionControl(), at.getValueType()));
    55       col.setFormatter(new CollectionFormatter(col.getExportFormatter()));
    56     }
    57     else
    58     {
    59       String path = tmp[3];
    60       PropertyPath property = null;
    61       if (!".".equals(path)) // special case '.' is equal to the "parent" item
     45      String[] tmp = expr.split("/");
     46      Item parentType = Item.valueOf(tmp[1]);
     47      ItemSubtype subtype = ItemSubtype.getById(dc, Values.getInt(tmp[2]));
     48      Filter<Annotatable> parentFilter = new SubtypeFilter(subtype);
     49      if (tmp[3].startsWith("#"))
    6250      {
    63         Metadata m = Metadata.getInstance(parentType.getItemClass());
    64         property = m.getPropertyPath(path, false);
    65         col = new ParentItemPropertyColumn(expr, parentFilter, helper, property);
    66         col.setTitle(subtype.getName()+"."+path);
    67         col.setTooltip(subtype.getName() + " › "+path);
     51        AnnotationType at = AnnotationType.getById(dc, Values.getInt(tmp[3].substring(1)));
     52        col = new ParentItemAnnotationColumn(expr, parentFilter, helper, at);
     53        col.setTitle(at.getName() + " [A]");
     54        col.setTooltip(subtype.getName() + " › "+at.getName());
     55        col.setValueType(at.getValueType());
     56        col.setExportFormatter(FormatterFactory.getTypeFormatter(dc.getSessionControl(), at.getValueType()));
     57        col.setFormatter(new CollectionFormatter(col.getExportFormatter()));
    6858      }
    6959      else
    7060      {
    71         col = new ParentItemPropertyColumn(expr, parentFilter, helper, null);
    72         col.setTitle(subtype.getName());
    73         col.setTooltip(subtype.getName());
    74       }
    75       if (property == null || property.getHibernateType().isEntityType())
    76       {
    77         col.setExportFormatter(new NameableFormatter());
    78         col.setFormatter(new CollectionFormatter(new LinkedItemFormatter()));
    79       }
    80       else
    81       {
    82         col.setExportFormatter(new ToStringFormatter<>());
    83         col.setFormatter(new CollectionFormatter(col.getExportFormatter()));
     61        String path = tmp[3];
     62        PropertyPath property = null;
     63        if (!".".equals(path)) // special case '.' is equal to the "parent" item
     64        {
     65          Metadata m = Metadata.getInstance(parentType.getItemClass());
     66          property = m.getPropertyPath(path, false);
     67          col = new ParentItemPropertyColumn(expr, parentFilter, helper, property);
     68          col.setTitle(subtype.getName()+"."+path);
     69          col.setTooltip(subtype.getName() + " › "+path);
     70        }
     71        else
     72        {
     73          col = new ParentItemPropertyColumn(expr, parentFilter, helper, null);
     74          col.setTitle(subtype.getName());
     75          col.setTooltip(subtype.getName());
     76        }
     77        if (property == null || property.getHibernateType().isEntityType())
     78        {
     79          col.setExportFormatter(new NameableFormatter());
     80          col.setFormatter(new CollectionFormatter(new LinkedItemFormatter()));
     81        }
     82        else
     83        {
     84          col.setExportFormatter(new ToStringFormatter<>());
     85          col.setFormatter(new CollectionFormatter(col.getExportFormatter()));
     86        }
    8487      }
    8588    }
     89    catch (RuntimeException ex)
     90    {}
    8691   
    8792    return col;
    … …  
    8994
    9095  private final Filter<Annotatable> filter;
    91   protected final ParentItemHelper helper;
     96  private final ParentItemHelper helper;
    9297 
    9398  ParentItemColumn(String id, Filter<Annotatable> filter, ParentItemHelper helper)
    … …  
    100105  }
    101106
    102   protected Set<Annotatable> getParentItems(DbControl dc, Annotatable item)
     107  /**
     108    We finalize this implementation to make sure that the helper
     109    implementation always get a chance to re-cycle transactions.
     110    Subclasses should implement {@link #getValue(ParentItemHelper, Annotatable)}
     111    to return the requested value from the item and use the
     112    DbControl from {@link ParentItemHelper#dc}.
     113  */
     114  @Override
     115  public final Object getValue(DbControl dc, Annotatable item)
    103116  {
    104     return AnnotationUtil.getAllAnnotatableParentItems(dc, item, filter, helper.cache);
     117    item = helper.recycle(dc, item);
     118    return getValue(helper, item);
     119  }
     120
     121  /**
     122    Alternate implementation for loading data.
     123  */
     124  protected abstract Object getValue(ParentItemHelper helper, Annotatable item);
     125 
     126  /**
     127    Get all parent items to the requested item that mathces the specified filter.
     128  */
     129  protected Set<Annotatable> getParentItems(Annotatable item)
     130  {
     131    return AnnotationUtil.getAllAnnotatableParentItems(helper.dc, item, filter, helper.cache);
    105132  }
    106133 
    107  
    108134}

  • trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/ParentItemColumnActionFactory.java

    r7765 r7767  
    77import net.sf.basedb.clients.web.extensions.JspContext;
    88import net.sf.basedb.core.Annotatable;
     9import net.sf.basedb.core.DbControl;
    910import net.sf.basedb.core.Item;
    1011import net.sf.basedb.core.ItemContext;
    … …  
    7475    if (allColumns == null || !allColumns.contains("/")) return null;
    7576   
    76     ParentItemHelper helper = new ParentItemHelper();
     77    DbControl dc = jspContext.getDbControl();
     78    ParentItemHelper helper = new ParentItemHelper(sc);
    7779    List<ListColumnAction<Annotatable, Object>> actions = new ArrayList<>();
    7880    for (String col : allColumns.split(","))
    … …  
    8082      if (col.startsWith("/"))
    8183      {
    82         ListColumnAction<Annotatable, Object> action = ParentItemColumn.create(jspContext.getDbControl(), col, helper);
     84        ListColumnAction<Annotatable, Object> action = ParentItemColumn.create(dc, col, helper);
    8385        if (action != null) actions.add(action);
    8486      }
    … …  
    8688   
    8789    if (actions.size() == 0) return null;
     90   
    8891    return actions.toArray(new ListColumnAction[actions.size()]);
    8992  }

  • trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/ParentItemHelper.java

    r7765 r7767  
    11package net.sf.basedb.clients.web.extensions.list;
    22
     3import java.util.HashSet;
     4import java.util.Set;
     5
     6import net.sf.basedb.core.Annotatable;
     7import net.sf.basedb.core.DbControl;
     8import net.sf.basedb.core.SessionControl;
     9import net.sf.basedb.core.TransactionalAction;
    310import net.sf.basedb.core.snapshot.SnapshotManager;
    411import net.sf.basedb.util.AnnotationUtil;
    … …  
    1421{
    1522
     23  private static final int MAX_ITEMS_IN_TRANSACTION = 200;
     24 
     25  final SessionControl sc;
    1626  // Use the snapshot manager for efficient loading of annotations
    1727  final SnapshotManager manager;
    1828  // Use the cache to improve performance of AnnotationUtil.getAllAnnotatableParents()
    1929  final AnnotationUtil.Cache cache;
     30  private final Set<Integer> items;
     31
     32  DbControl dc;
    2033 
    21   ParentItemHelper()
     34  ParentItemHelper(SessionControl sc)
    2235  {
    23     manager = new SnapshotManager();
    24     cache = AnnotationUtil.createCache();
     36    this.sc = sc;
     37    this.manager = new SnapshotManager();
     38    this.cache = AnnotationUtil.createCache();
     39    this.items = new HashSet<>();
    2540  }
    2641 
     42  /**
     43    Re-cycle transactions if needed and re-load the given item
     44    in the new transaction.
     45  */
     46  Annotatable recycle(DbControl dcExt, Annotatable item)
     47  {
     48    if (dc == null) dc = dcExt;
     49
     50    int itemId = item.getId();
     51    if (items.add(itemId) && items.size() > MAX_ITEMS_IN_TRANSACTION)
     52    {
     53      // For every MAX_ITEMS_IN_TRANSACTION items, we create a new DbControl
     54      // in order to let garbage collection clean up alread processed items.
     55      // Otherwise we risk that the "entire" database is loaded into
     56      // a single DbControl
     57      if (dc != dcExt)
     58      {
     59        // Close the old DbControl but only if it is one that we created
     60        dc.close();
     61      }
     62      else
     63      {
     64        // This makes sure that our last DbControl is closed when the
     65        // external transaction is completed
     66        dcExt.addTransactionalAction(new CloseMyDbControl(this));
     67      }
     68     
     69      dc = sc.newDbControl();
     70      cache.clear();
     71      manager.clear();
     72      items.clear();
     73      items.add(itemId);
     74    }
     75   
     76    // We reload the item in the current DbControl
     77    return item.getType().getById(dc, itemId);
     78  }
     79 
     80  // Close and cleanup
     81  void close()
     82  {
     83    if (dc != null) dc.close();
     84    manager.clear();
     85    cache.clear();
     86    items.clear();
     87  }
     88 
     89  static class CloseMyDbControl
     90    implements TransactionalAction
     91  {
     92   
     93    private final ParentItemHelper helper;
     94    public CloseMyDbControl(ParentItemHelper helper)
     95    {
     96      this.helper = helper;
     97    }
     98   
     99    @Override
     100    public void onRollback()
     101    {
     102      helper.close();
     103    }
     104   
     105    @Override
     106    public void onBeforeCommit()
     107    {
     108      helper.close();
     109    }
     110   
     111    @Override
     112    public void onAfterCommit()
     113    {}
     114   
     115  }
     116 
     117 
    27118}

  • trunk/src/clients/web/net/sf/basedb/clients/web/extensions/list/ParentItemPropertyColumn.java

    r7766 r7767  
    55
    66import net.sf.basedb.core.Annotatable;
    7 import net.sf.basedb.core.DbControl;
     7import net.sf.basedb.core.Metadata;
    88import net.sf.basedb.core.Metadata.PropertyPath;
    99import net.sf.basedb.util.filter.Filter;
    1010
     11/**
     12  Parent item column that get values from parent items by specifying
     13  a "property path". See {@link Metadata}.
     14
     15  @author nicklas
     16  @since 3.16
     17*/
    1118public class ParentItemPropertyColumn
    1219  extends ParentItemColumn
    1320{
    1421
    15   private final PropertyPath property;
     22  private final PropertyPath<? super Annotatable, ?> property;
    1623 
    17   ParentItemPropertyColumn(String id, Filter<Annotatable> filter, ParentItemHelper helper, PropertyPath property)
     24  ParentItemPropertyColumn(String id, Filter<Annotatable> filter, ParentItemHelper helper, PropertyPath<? super Annotatable, ?> property)
    1825  {
    1926    super(id, filter, helper);
    … …  
    2229 
    2330  @Override
    24   public Object getValue(DbControl dc, Annotatable item)
     31  public Object getValue(ParentItemHelper helper, Annotatable item)
    2532  {
    26     Set<Annotatable> parents = getParentItems(dc, item);
     33    Set<Annotatable> parents = getParentItems(item);
    2734    if (parents.size() == 0) return null;
    2835
    … …  
    3037    for (Annotatable p : parents)
    3138    {
    32       values.add(property == null ? p : property.getValue(dc, p));
     39      values.add(property == null ? p : property.getValue(helper.dc, p));
    3340    }
    3441   

  • trunk/src/core/net/sf/basedb/core/snapshot/SnapshotManager.java

    r7610 r7767  
    117117  {
    118118    this.snapshots = new HashMap<Integer, AnnotationSetSnapshot>();
     119  }
     120 
     121  /**
     122    Clear the manager from all memory-cached information.
     123    @since 3.16
     124  */
     125  public void clear()
     126  {
     127    snapshots.clear();
    119128  }
    120129 

  • trunk/src/core/net/sf/basedb/util/AnnotationUtil.java

    r7002 r7767  
    332332      this.cmap = new HashMap<AnnotatableWrapper, Set<AnnotatableWrapper>>();
    333333    }
     334 
     335    /**
     336      Clear all information from the cache.
     337      @since 3.16
     338    */
     339    public void clear()
     340    {
     341      cmap.clear();
     342    }
    334343   
    335344    boolean contains(AnnotatableWrapper wrapper)
Note: See TracChangeset for help on using the changeset viewer.