Changeset 3650

Timestamp:

Aug 10, 2007, 11:46:50 AM (16 years ago)

Author:

Nicklas Nordborg

Message:

Fixed performance problem with reporter migration.
Added support for BASE 1.2.21 schema version.

Location:

trunk/src/clients/migrate/net/sf/basedb/clients/migrate

Files:

• Migrater.java (modified) (3 diffs)
• ReporterTransfer.java (modified) (6 diffs)

trunk/src/clients/migrate/net/sf/basedb/clients/migrate/Migrater.java

@@ -38 +38 @@
 import java.sql.SQLException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Properties;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -52 +54 @@
 public class Migrater extends Manager
 {
+  
+  public static Set<String> supportedBase1Versions = new HashSet<String>();
+  static
+  {
+    supportedBase1Versions.add("1.2.20");
+    supportedBase1Versions.add("1.2.21");
+  }
+  
   /**
   Holds properties defined in migrate.properties
@@ -431 +441 @@
       String schemaVersion = rs.getString(2);
       ps.close();
-      if (!schemaVersion.equals("1.2.20"))
+      if (!supportedBase1Versions.contains(schemaVersion))
       {
         log.error("Migration cannot proceed due to incompatible schema version. Your schema version is "
-            + schemaVersion + " and this migration needs 1.2.20");
+            + schemaVersion + " and this migration needs one of: " + supportedBase1Versions);
         stop();
       }     

trunk/src/clients/migrate/net/sf/basedb/clients/migrate/ReporterTransfer.java

@@ -105 +105 @@
     
     int count = getInt("countReporters");
-    startProgress(count*2, "Reporters"); // First pass creates reporters, second one reads in the new ID:s
+    // First pass creates reporters...
+    startProgress(count, "Reporters"); 
     PreparedStatement ps = QUICK_TRANSFER ? prepareStatementFromFile("selectMinimumReporters")
         : prepareStatementFromFile("selectReporters");
@@ -111 +112 @@
     externalIdMap = new HashMap<String, Integer>(count);
     duplicates = new HashMap<Integer, Integer>();
-    DbControl dc = newDbControl();
+    DbControl dc = null;
     try
     {
+      dc = newDbControl();
       batcher = ReporterBatcher.getNew(dc);
       batcher.setBatchSize(getBatchSize());
@@ -119 +121 @@
       batcher.close();
       dc.commit();
-      
+    }
+    finally
+    {
+      if (dc != null) dc.close();
+      progress.stop();
+    }
+    
+    // ... second pass reads in the new ID:s
+    progress = null;
+    startProgress(count, "Mapping reporter ID:s"); 
+    try
+    {
       reporterMap = new HashMap<Integer, ReporterData>(count);
+      dc = newDbControl();
       log.info("Mapping reporters in BASE 2...");
-      mapReporters(0, getBatchSize());
+      int start = 0;
+      int batchSize = getBatchSize();
+
+      DataQuery<ReporterData> query = Reporter.getQuery();
+      query.order(Orders.asc(Hql.property("id")));
+      query.setMaxResults(batchSize);
+      while (start < count)
+      {
+        query.setFirstResult(start);
+        cleanUpMemory();
+        mapMigratedReporters(dc, query);
+        start += batchSize;
+      }
       cleanUpMemory();
+      mapDuplicateReporters();
     }
     finally
@@ -189 +216 @@
   }
 
-  private void mapReporters(int from, int size)
-  {
-    cleanUpMemory();
-    int rowCount = 0;
-    DbControl dc = newDbControl();
+  private void mapMigratedReporters(DbControl dc, DataQuery<ReporterData> query)
+  {
     DataResultIterator<ReporterData> result = null;
     try
     {
-      DataQuery<ReporterData> query = Reporter.getQuery();
-      query.order(Orders.asc(Hql.property("id")));
-      query.setFirstResult(from);
-      query.setMaxResults(size);
       result = query.iterate(dc);
       while (result.hasNext())
@@ -207 +227 @@
         Integer base1Id = externalIdMap.get(reporter.getExternalId());
         reporterMap.put(base1Id, new ReporterProxy(reporter));
-        rowCount++;
         progress.increase();
         if (log.isDebugEnabled())
@@ -215 +234 @@
         }
       }
-      
-      // Map duplicates
-      for (Map.Entry<Integer, Integer> entry : duplicates.entrySet())
-      {
-        Integer base1Id = entry.getKey();
-        Integer migratedBase1Id = entry.getValue();
-        ReporterData migratedReporter = reporterMap.get(migratedBase1Id);
-        reporterMap.put(base1Id, migratedReporter);
-        if (log.isDebugEnabled())
-        {
-          log.debug("Reporter: " + migratedReporter.getExternalId() + 
-            " has been mapped to BASE 1 ID=" + base1Id);
-        }
-      }
     }
     finally
     {
       if (result != null) result.close();
-      if (dc != null) dc.close();
-    }
-    if (rowCount == size)
-    {
-      mapReporters(from+size, size);
-    }
-  }
-
+    }
+  }
+
+  private void mapDuplicateReporters()
+  {
+    // The map contains base1Id of duplicate -> base1Id of transfered reporter
+    for (Map.Entry<Integer, Integer> entry : duplicates.entrySet())
+    {
+      Integer duplicateId = entry.getKey();
+      Integer migratedId = entry.getValue();
+      ReporterData migratedReporter = reporterMap.get(migratedId);
+      reporterMap.put(duplicateId, migratedReporter);
+      progress.increase();
+      if (log.isDebugEnabled())
+      {
+        log.debug("Reporter: " + migratedReporter.getExternalId() + 
+          " has been mapped to BASE 1 ID=" + duplicateId);
+      }
+    }
+  }
+  
+  
 
   /**