Changeset 3709

Timestamp:

Sep 6, 2007, 9:28:40 AM (16 years ago)

Author:

Nicklas Nordborg

Message:

References #744: Make it possible to resume the migration program in case of a failure

It should now be possible to resume the migration if there is a failure during raw data transfer.

Location:

branches/2.4-stable

Files:

• config/dist/mysql-migration-queries.sql (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/CommonItemTransfer.java (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/GroupTransfer.java (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/Migrater.java (modified) (11 diffs)
• src/clients/migrate/net/sf/basedb/clients/migrate/QuotaTransfer.java (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayDataTransfer.java (modified) (5 diffs)
• src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayTransfer.java (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/ReporterTransfer.java (modified) (4 diffs)
• src/clients/migrate/net/sf/basedb/clients/migrate/SharedItemTransfer.java (modified) (1 diff)
• src/clients/migrate/net/sf/basedb/clients/migrate/Transfer.java (modified) (2 diffs)
• src/clients/migrate/net/sf/basedb/clients/migrate/UserTransfer.java (modified) (3 diffs)

branches/2.4-stable/config/dist/mysql-migration-queries.sql

@@ -1126 +1126 @@
 
 /* selectMinimumReporters */
-SELECT `id`, `reporterId` 
+SELECT `id`, `reporterId`, `geneName`, `geneSymbol`
 FROM `Reporter` 
 ORDER BY `id` 

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/CommonItemTransfer.java

@@ -62 +62 @@
   private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.CommonItemTransfer");
   
+  CommonItemTransfer()
+  {}
   
   CommonItemTransfer(UserTransfer userT, GroupTransfer groupT)

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/GroupTransfer.java

@@ -68 +68 @@
   private QuotaTransfer quotaT = null;
 
+  GroupTransfer()
+  {}
+  
   /**
   @param userT Depends on a finnished UserTransfer.

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/Migrater.java

@@ -31 +31 @@
 import net.sf.basedb.core.SessionControl;
 import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -99 +103 @@
     try
     {
-      ReporterTransfer reporterT = new ReporterTransfer();
+      ReporterTransfer reporterT = new ReporterTransfer(false);
       startTransfer(reporterT, "Reporter");
       RoleTransfer roleT = new RoleTransfer();
@@ -108 +112 @@
       startTransfer(userT, "User");
       GroupTransfer groupT = new GroupTransfer(userT, quotaT);
-      startTransfer(groupT, "Group");
+      startTransfer(groupT, "Group");   
       PluginTransfer pluginT = new PluginTransfer(userT, groupT);
       startTransfer(pluginT, "Plugins");
@@ -189 +193 @@
           groupT, scanT, arrayTypeT, uploadFT, softwareT, sampleT);
       startTransfer(rawBioAssayT, "RawBioAssay");
+      createResumePoint(userT, groupT, rawBioAssayT);
       RawBioAssayDataTransfer rawBioAssayDataT = new RawBioAssayDataTransfer(
-          userT, groupT, rawBioAssayT, arrayTypeT, reporterT);
+          rawBioAssayT, reporterT);
       startTransfer(rawBioAssayDataT, "RawBioAssayData");
+      removeResumePoint();
       ExperimentTransfer experimentT = new ExperimentTransfer(userT,
           groupT, rawBioAssayT);
@@ -203 +209 @@
       System.exit(1);
     }
-    catch (Exception e)
+    catch (Throwable e)
     {
       log.error("The migration FAILED", e);
@@ -210 +216 @@
   }
 
+  public void resume()
+  {
+    try
+    {
+      // Reporters must be mapped using external ID from BASE 1 and BASE 2
+      ReporterTransfer reporterT = new ReporterTransfer(true);
+      startTransfer(reporterT, "Reporter");
+      
+      // Pre-existing quota is loaded from BASE 2
+      QuotaTransfer quotaT = new QuotaTransfer();
+      startTransfer(quotaT, "Quota");
+
+      ObjectInputStream ois = new ObjectInputStream(new FileInputStream("migration.tmp"));
+      // Map of user ID:s are read from file
+      UserTransfer userT = new UserTransfer(quotaT);
+      userT.setup(FROM, TO, sessionControl, BASE2, migrateProperties);
+      log.info("Loading users from 'migration.tmp' file...");
+      userT.readRestorePointData(ois);
+      log.info("Found " + userT.getIdMap().size() + " users");
+      if (log.isDebugEnabled()) log.debug("User ID Map (Base 1 --> Base2): " + userT.getIdMap());
+
+      // Map of group ID:s are read from file
+      GroupTransfer groupT = new GroupTransfer();
+      log.info("Loading groups from 'migration.tmp' file...");
+      groupT.readRestorePointData(ois);
+      log.info("Found " + groupT.getIdMap().size() + " groups");
+      if (log.isDebugEnabled()) log.debug("Group ID Map (Base 1 --> Base2): " + groupT.getIdMap());
+      
+      // Map of raw bioassay ID:s are read from file
+      RawBioAssayTransfer rawBioAssayT = new RawBioAssayTransfer();
+      log.info("Loading raw bioassays from 'migration.tmp' file...");
+      rawBioAssayT.readRestorePointData(ois);
+      log.info("Found " + rawBioAssayT.getIdMap().size() + " raw bioassays");
+      if (log.isDebugEnabled()) log.debug("Raw bioassay ID Map (Base 1 --> Base2): " + rawBioAssayT.getIdMap());
+      ois.close();
+      
+      // RawBioassayTransfer runs normally, but checks if a raw bioassay already has raw data
+      RawBioAssayDataTransfer rawBioAssayDataT = new RawBioAssayDataTransfer(rawBioAssayT, 
+          reporterT);
+      startTransfer(rawBioAssayDataT, "RawBioAssayData");
+      removeResumePoint();
+      
+      // Experiment transfter runs normally
+      ExperimentTransfer experimentT = new ExperimentTransfer(userT,
+          groupT, rawBioAssayT);
+      startTransfer(experimentT, "Experiment");
+      
+      // Runs normally
+      userT.finalizeQuota();
+
+      print("END MIGRATION");
+      log.info("Migration ended");
+      Application.stop();
+      System.exit(1);
+      
+    }
+    catch (Throwable e)
+    {
+      log.error("The migration FAILED", e);
+      stop();
+    }
+  }
+  
+  private void createResumePoint(Transfer...transfers)
+    throws IOException
+  {
+    log.info("Creating resume point");
+    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream("migration.tmp"));
+    for (Transfer t: transfers)
+    {
+      log.info("Store to resume point: " + t);
+      t.writeRestorePointData(oos);
+    }
+    oos.close();
+  }
+  
+  private void removeResumePoint()
+  {
+    log.info("Removing resume point");
+    java.io.File resumePoint = new java.io.File("migration.tmp");
+    resumePoint.delete();
+  }
+  
   private void startTransfer(Transfer transfer, String property)
   {
@@ -474 +563 @@
     System.err.println("-----------------------------------------------------------------------");
 
+    java.io.File restorePoint = new java.io.File("migration.tmp");
+    boolean resumeMigration = false;
+
     if (args == null || args.length == 0 || !"disclaimer_understood".equals(args[0]))
     {
@@ -479 +571 @@
       System.exit(0);
     }
+    else if (restorePoint.exists())
+    {
+      System.err.println("FOUND DATA FROM A PREVIOUSLY INTERRUPTED MIGRATION.");
+      System.err.println("\n   Press ENTER to resume the migration\nor\n   Ctrl-C to abort\n");
+      System.err.println("To start a new migration delete the './migration.tmp' file");
+      System.err.println("before starting the migration program.");
+      waitForEnter();
+      resumeMigration = true;
+    }
     else
     {
@@ -485 +586 @@
     }
     
-    System.err.println("Starting migration...");
+    System.err.println(resumeMigration ? "Resuming migration" : "Starting migration...");
     Application.start(false);
     Migrater migrater = new Migrater();
@@ -493 +594 @@
     migrater.prepareStatementsFromFile();
     log.info("Migration initialisation DONE");
-    migrater.run();
+    if (resumeMigration)
+    {
+      migrater.resume();
+    }
+    else
+    {
+      migrater.run();
+    }
   }
   
@@ -506 +614 @@
     {}
   }
-  
-  
+    
 }

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/QuotaTransfer.java

@@ -95 +95 @@
       quota.setName(formatBytes(totalQuota));
       dc.saveItem(quota);
+      quotas.put(totalQuota, quota);
     }
     else

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayDataTransfer.java

@@ -47 +47 @@
 
 public class RawBioAssayDataTransfer 
-  extends CommonItemTransfer
+  extends Transfer
 {
   /**
@@ -77 +77 @@
   @param groupT A finished {@link GroupTransfer}
    */
-  RawBioAssayDataTransfer(UserTransfer userT, GroupTransfer groupT,
-      RawBioAssayTransfer rbaT, ArrayTypeTransfer arrayTypeT,
-      ReporterTransfer reporterT)
-  {
-    super(userT, groupT);
+  RawBioAssayDataTransfer(RawBioAssayTransfer rbaT, ReporterTransfer reporterT)
+  {
+    super();
     log.debug("Checking that necessary transfers have been done before proceeding with RawBioAssayData");
-    if (rbaT != null && rbaT.isDone() && arrayTypeT != null
-        && arrayTypeT.isDone() && reporterT != null
-        && reporterT.isDone())
+    if (rbaT != null && rbaT.isDone() && 
+      reporterT != null && reporterT.isDone())
     {
       this.rbaT = rbaT;
@@ -93 +90 @@
     {
       if (!rbaT.isDone()) log.debug("Dependency: RawBioAssays haven't been transfered");
-      if (!arrayTypeT.isDone()) log.debug("Dependency: ArrayTypes haven't been transfered");
       if (!reporterT.isDone()) log.debug("Dependency: Reporters haven't been transfered");
       dependencyFailure();
@@ -133 +129 @@
         log.debug("RawBioAssay '" + rawBioAssayId + "' " + (hasArrayDesign ? "has " : "no ") + "Array Design");
       }
-      transferRawBioAssayData(rawBioAssay, rawBioAssayId, hasArrayDesign);
+      if (!rawBioAssay.hasData())
+      {
+        transferRawBioAssayData(rawBioAssay, rawBioAssayId, hasArrayDesign);
+      }
+      else
+      {
+        log.info("Skipping since it already has data");
+        progress.increase(rawBioAssay.getSpots());
+      }
       progress.decrease(); // We do not want to count the RawBioAssays
       if (log.isDebugEnabled())
@@ -263 +267 @@
       if (log.isInfoEnabled())
       {
-        log.info("Transfered " + rowCount + " rows of RawData for raw bioassay" + rawBioAssay.getName());
+        log.info("Transfered " + rowCount + " rows of RawData for raw bioassay: " + rawBioAssay.getName());
       }
     }

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayTransfer.java

@@ -63 +63 @@
   private SampleTransfer sampleT = null;
 
+  RawBioAssayTransfer()
+  {}
+  
   /**
   Constructs a new RawBioAssayTransfer.

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/ReporterTransfer.java

@@ -90 +90 @@
   
   /**
+    Flag that indicates if we should run in RESUME mode 
+    (ie. NOT create new reporter items, only map ID:s)
+  */
+  private boolean RESUME = false;
+  
+  /**
   Set string and text properties, order must match the select statement
   except for the id and reporterId in the beginning Extended properties are
@@ -102 +108 @@
   Reporters are independant from other transfers.
    */
-  ReporterTransfer()
-  {}
+  ReporterTransfer(boolean resume)
+  {
+    RESUME = resume;
+  }
 
   public void start()
   {
-    if (!QUICK_TRANSFER) log.info("Transferring reporters...");
-    else log.info("Transferring reporters with minimum properties...");
+    if (RESUME) QUICK_TRANSFER = true;
+    if (RESUME)
+    {
+      log.info("Loading reporters...");
+    }
+    else if (!QUICK_TRANSFER)
+    {
+      log.info("Transferring reporters...");
+    }
+    else 
+    {
+      log.info("Transferring reporters with minimum properties...");
+    }
     
     int count = getInt("countReporters");
@@ -199 +218 @@
         rowCount++;
         ReporterData item = createDataItem(rs);
-        if (item != null) batcher.insert(item);
+        if (item != null && !RESUME) batcher.insert(item);
         progress.increase();
       }
@@ -319 +338 @@
       }
       rd = Reporter.getNew(externalId);
+      String name = rs.getString(3);
+      if (name == null || "".equals(name)) name = externalId;
+      rd.setName(name);
+      rd.setSymbol(rs.getString(4));
       if (!QUICK_TRANSFER)
       {
-        String name = rs.getString(3);
-        if (name == null || "".equals(name)) name = externalId;
-        rd.setName(name);
-        rd.setSymbol(rs.getString(4));
         int columnIndex = 5;
         for (String property : stringProperties)

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/SharedItemTransfer.java

@@ -83 +83 @@
   private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.SharedItemTransfer");
 
+  SharedItemTransfer()
+  {}
+  
   SharedItemTransfer(UserTransfer userT, GroupTransfer groupT)
   {

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/Transfer.java

@@ -32 +32 @@
 import net.sf.basedb.core.ItemAlreadyExistsException;
 import net.sf.basedb.core.SessionControl;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -613 +617 @@
     return Integer.parseInt(value);
   }
+  
+  /**
+    Store ID map and some boolean flags
+  */
+  protected void writeRestorePointData(ObjectOutputStream out)
+    throws IOException
+  {
+    out.writeBoolean(dependencyFailed);
+    out.writeBoolean(done);
+    out.writeBoolean(idMap != null);
+    if (idMap != null) out.writeObject(idMap);
+  }
+  
+  /**
+    Store ID map and some boolean flags
+  */
+  @SuppressWarnings("unchecked")
+  protected void readRestorePointData(ObjectInputStream in)
+    throws IOException, ClassNotFoundException
+  {
+    dependencyFailed = in.readBoolean();
+    done = in.readBoolean();
+    if (in.readBoolean())
+    {
+      idMap = (Map<Integer, Integer>)in.readObject();
+    }
+  }
+
 }

branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/UserTransfer.java

@@ -36 +36 @@
 import net.sf.basedb.core.SystemItems;
 import net.sf.basedb.core.User;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -90 +94 @@
    */
   private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.UserTransfer");
+  
+  UserTransfer(QuotaTransfer quotaT)
+  {
+    this.quotaT = quotaT;
+  }
   
   /**
@@ -323 +332 @@
   }
   
+  /**
+    Store quota map
+  */
+  protected void writeRestorePointData(ObjectOutputStream out)
+    throws IOException
+  {
+    super.writeRestorePointData(out);
+    out.writeObject(base1FileQuota);
+  }
+  
+  /**
+    Store ID map and some boolean flags
+  */
+  @SuppressWarnings("unchecked")
+  protected void readRestorePointData(ObjectInputStream in)
+    throws IOException, ClassNotFoundException
+  {
+    super.readRestorePointData(in);
+    base1FileQuota = (Map<Integer, Long>)in.readObject();
+  }
+  
   public User getUserById(DbControl dc, int base1Id)
   {