Changeset 3650


Ignore:
Timestamp:
Aug 10, 2007, 11:46:50 AM (14 years ago)
Author:
Nicklas Nordborg
Message:

Fixed performance problem with reporter migration.
Added support for BASE 1.2.21 schema version.

Location:
trunk/src/clients/migrate/net/sf/basedb/clients/migrate
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/src/clients/migrate/net/sf/basedb/clients/migrate/Migrater.java

    r3529 r3650  
    3838import java.sql.SQLException;
    3939import java.util.HashMap;
     40import java.util.HashSet;
    4041import java.util.Properties;
     42import java.util.Set;
    4143import java.util.regex.Matcher;
    4244import java.util.regex.Pattern;
     
    5254public class Migrater extends Manager
    5355{
     56 
     57  public static Set<String> supportedBase1Versions = new HashSet<String>();
     58  static
     59  {
     60    supportedBase1Versions.add("1.2.20");
     61    supportedBase1Versions.add("1.2.21");
     62  }
     63 
    5464  /**
    5565  Holds properties defined in migrate.properties
     
    431441      String schemaVersion = rs.getString(2);
    432442      ps.close();
    433       if (!schemaVersion.equals("1.2.20"))
     443      if (!supportedBase1Versions.contains(schemaVersion))
    434444      {
    435445        log.error("Migration cannot proceed due to incompatible schema version. Your schema version is "
    436             + schemaVersion + " and this migration needs 1.2.20");
     446            + schemaVersion + " and this migration needs one of: " + supportedBase1Versions);
    437447        stop();
    438448      }     
  • trunk/src/clients/migrate/net/sf/basedb/clients/migrate/ReporterTransfer.java

    r3634 r3650  
    105105   
    106106    int count = getInt("countReporters");
    107     startProgress(count*2, "Reporters"); // First pass creates reporters, second one reads in the new ID:s
     107    // First pass creates reporters...
     108    startProgress(count, "Reporters");
    108109    PreparedStatement ps = QUICK_TRANSFER ? prepareStatementFromFile("selectMinimumReporters")
    109110        : prepareStatementFromFile("selectReporters");
     
    111112    externalIdMap = new HashMap<String, Integer>(count);
    112113    duplicates = new HashMap<Integer, Integer>();
    113     DbControl dc = newDbControl();
     114    DbControl dc = null;
    114115    try
    115116    {
     117      dc = newDbControl();
    116118      batcher = ReporterBatcher.getNew(dc);
    117119      batcher.setBatchSize(getBatchSize());
     
    119121      batcher.close();
    120122      dc.commit();
    121      
     123    }
     124    finally
     125    {
     126      if (dc != null) dc.close();
     127      progress.stop();
     128    }
     129   
     130    // ... second pass reads in the new ID:s
     131    progress = null;
     132    startProgress(count, "Mapping reporter ID:s");
     133    try
     134    {
    122135      reporterMap = new HashMap<Integer, ReporterData>(count);
     136      dc = newDbControl();
    123137      log.info("Mapping reporters in BASE 2...");
    124       mapReporters(0, getBatchSize());
     138      int start = 0;
     139      int batchSize = getBatchSize();
     140
     141      DataQuery<ReporterData> query = Reporter.getQuery();
     142      query.order(Orders.asc(Hql.property("id")));
     143      query.setMaxResults(batchSize);
     144      while (start < count)
     145      {
     146        query.setFirstResult(start);
     147        cleanUpMemory();
     148        mapMigratedReporters(dc, query);
     149        start += batchSize;
     150      }
    125151      cleanUpMemory();
     152      mapDuplicateReporters();
    126153    }
    127154    finally
     
    189216  }
    190217
    191   private void mapReporters(int from, int size)
    192   {
    193     cleanUpMemory();
    194     int rowCount = 0;
    195     DbControl dc = newDbControl();
     218  private void mapMigratedReporters(DbControl dc, DataQuery<ReporterData> query)
     219  {
    196220    DataResultIterator<ReporterData> result = null;
    197221    try
    198222    {
    199       DataQuery<ReporterData> query = Reporter.getQuery();
    200       query.order(Orders.asc(Hql.property("id")));
    201       query.setFirstResult(from);
    202       query.setMaxResults(size);
    203223      result = query.iterate(dc);
    204224      while (result.hasNext())
     
    207227        Integer base1Id = externalIdMap.get(reporter.getExternalId());
    208228        reporterMap.put(base1Id, new ReporterProxy(reporter));
    209         rowCount++;
    210229        progress.increase();
    211230        if (log.isDebugEnabled())
     
    215234        }
    216235      }
    217      
    218       // Map duplicates
    219       for (Map.Entry<Integer, Integer> entry : duplicates.entrySet())
    220       {
    221         Integer base1Id = entry.getKey();
    222         Integer migratedBase1Id = entry.getValue();
    223         ReporterData migratedReporter = reporterMap.get(migratedBase1Id);
    224         reporterMap.put(base1Id, migratedReporter);
    225         if (log.isDebugEnabled())
    226         {
    227           log.debug("Reporter: " + migratedReporter.getExternalId() +
    228             " has been mapped to BASE 1 ID=" + base1Id);
    229         }
    230       }
    231236    }
    232237    finally
    233238    {
    234239      if (result != null) result.close();
    235       if (dc != null) dc.close();
    236     }
    237     if (rowCount == size)
    238     {
    239       mapReporters(from+size, size);
    240     }
    241   }
    242 
     240    }
     241  }
     242
     243  private void mapDuplicateReporters()
     244  {
     245    // The map contains base1Id of duplicate -> base1Id of transfered reporter
     246    for (Map.Entry<Integer, Integer> entry : duplicates.entrySet())
     247    {
     248      Integer duplicateId = entry.getKey();
     249      Integer migratedId = entry.getValue();
     250      ReporterData migratedReporter = reporterMap.get(migratedId);
     251      reporterMap.put(duplicateId, migratedReporter);
     252      progress.increase();
     253      if (log.isDebugEnabled())
     254      {
     255        log.debug("Reporter: " + migratedReporter.getExternalId() +
     256          " has been mapped to BASE 1 ID=" + duplicateId);
     257      }
     258    }
     259  }
     260 
     261 
    243262
    244263  /**
Note: See TracChangeset for help on using the changeset viewer.