Changeset 3709


Ignore:
Timestamp:
Sep 6, 2007, 9:28:40 AM (14 years ago)
Author:
Nicklas Nordborg
Message:

References #744: Make it possible to resume the migration program in case of a failure

It should now be possible to resume the migration if there is a failure during raw data transfer.

Location:
branches/2.4-stable
Files:
11 edited

Legend:

Unmodified
Added
Removed
  • branches/2.4-stable/config/dist/mysql-migration-queries.sql

    r3675 r3709  
    11261126
    11271127/* selectMinimumReporters */
    1128 SELECT `id`, `reporterId` 
     1128SELECT `id`, `reporterId`, `geneName`, `geneSymbol`
    11291129FROM `Reporter`
    11301130ORDER BY `id`
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/CommonItemTransfer.java

    r3679 r3709  
    6262  private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.CommonItemTransfer");
    6363 
     64  CommonItemTransfer()
     65  {}
    6466 
    6567  CommonItemTransfer(UserTransfer userT, GroupTransfer groupT)
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/GroupTransfer.java

    r3683 r3709  
    6868  private QuotaTransfer quotaT = null;
    6969
     70  GroupTransfer()
     71  {}
     72 
    7073  /**
    7174  @param userT Depends on a finnished UserTransfer.
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/Migrater.java

    r3679 r3709  
    3131import net.sf.basedb.core.SessionControl;
    3232import java.io.BufferedReader;
     33import java.io.FileInputStream;
     34import java.io.FileOutputStream;
    3335import java.io.IOException;
    3436import java.io.InputStream;
    3537import java.io.InputStreamReader;
     38import java.io.ObjectInputStream;
     39import java.io.ObjectOutputStream;
    3640import java.sql.Connection;
    3741import java.sql.DriverManager;
     
    99103    try
    100104    {
    101       ReporterTransfer reporterT = new ReporterTransfer();
     105      ReporterTransfer reporterT = new ReporterTransfer(false);
    102106      startTransfer(reporterT, "Reporter");
    103107      RoleTransfer roleT = new RoleTransfer();
     
    108112      startTransfer(userT, "User");
    109113      GroupTransfer groupT = new GroupTransfer(userT, quotaT);
    110       startTransfer(groupT, "Group");
     114      startTransfer(groupT, "Group");   
    111115      PluginTransfer pluginT = new PluginTransfer(userT, groupT);
    112116      startTransfer(pluginT, "Plugins");
     
    189193          groupT, scanT, arrayTypeT, uploadFT, softwareT, sampleT);
    190194      startTransfer(rawBioAssayT, "RawBioAssay");
     195      createResumePoint(userT, groupT, rawBioAssayT);
    191196      RawBioAssayDataTransfer rawBioAssayDataT = new RawBioAssayDataTransfer(
    192           userT, groupT, rawBioAssayT, arrayTypeT, reporterT);
     197          rawBioAssayT, reporterT);
    193198      startTransfer(rawBioAssayDataT, "RawBioAssayData");
     199      removeResumePoint();
    194200      ExperimentTransfer experimentT = new ExperimentTransfer(userT,
    195201          groupT, rawBioAssayT);
     
    203209      System.exit(1);
    204210    }
    205     catch (Exception e)
     211    catch (Throwable e)
    206212    {
    207213      log.error("The migration FAILED", e);
     
    210216  }
    211217
     218  public void resume()
     219  {
     220    try
     221    {
     222      // Reporters must be mapped using external ID from BASE 1 and BASE 2
     223      ReporterTransfer reporterT = new ReporterTransfer(true);
     224      startTransfer(reporterT, "Reporter");
     225     
     226      // Pre-existing quota is loaded from BASE 2
     227      QuotaTransfer quotaT = new QuotaTransfer();
     228      startTransfer(quotaT, "Quota");
     229
     230      ObjectInputStream ois = new ObjectInputStream(new FileInputStream("migration.tmp"));
     231      // Map of user ID:s are read from file
     232      UserTransfer userT = new UserTransfer(quotaT);
     233      userT.setup(FROM, TO, sessionControl, BASE2, migrateProperties);
     234      log.info("Loading users from 'migration.tmp' file...");
     235      userT.readRestorePointData(ois);
     236      log.info("Found " + userT.getIdMap().size() + " users");
     237      if (log.isDebugEnabled()) log.debug("User ID Map (Base 1 --> Base2): " + userT.getIdMap());
     238
     239      // Map of group ID:s are read from file
     240      GroupTransfer groupT = new GroupTransfer();
     241      log.info("Loading groups from 'migration.tmp' file...");
     242      groupT.readRestorePointData(ois);
     243      log.info("Found " + groupT.getIdMap().size() + " groups");
     244      if (log.isDebugEnabled()) log.debug("Group ID Map (Base 1 --> Base2): " + groupT.getIdMap());
     245     
     246      // Map of raw bioassay ID:s are read from file
     247      RawBioAssayTransfer rawBioAssayT = new RawBioAssayTransfer();
     248      log.info("Loading raw bioassays from 'migration.tmp' file...");
     249      rawBioAssayT.readRestorePointData(ois);
     250      log.info("Found " + rawBioAssayT.getIdMap().size() + " raw bioassays");
     251      if (log.isDebugEnabled()) log.debug("Raw bioassay ID Map (Base 1 --> Base2): " + rawBioAssayT.getIdMap());
     252      ois.close();
     253     
     254      // RawBioassayTransfer runs normally, but checks if a raw bioassay already has raw data
     255      RawBioAssayDataTransfer rawBioAssayDataT = new RawBioAssayDataTransfer(rawBioAssayT,
     256          reporterT);
     257      startTransfer(rawBioAssayDataT, "RawBioAssayData");
     258      removeResumePoint();
     259     
     260      // Experiment transfter runs normally
     261      ExperimentTransfer experimentT = new ExperimentTransfer(userT,
     262          groupT, rawBioAssayT);
     263      startTransfer(experimentT, "Experiment");
     264     
     265      // Runs normally
     266      userT.finalizeQuota();
     267
     268      print("END MIGRATION");
     269      log.info("Migration ended");
     270      Application.stop();
     271      System.exit(1);
     272     
     273    }
     274    catch (Throwable e)
     275    {
     276      log.error("The migration FAILED", e);
     277      stop();
     278    }
     279  }
     280 
     281  private void createResumePoint(Transfer...transfers)
     282    throws IOException
     283  {
     284    log.info("Creating resume point");
     285    ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream("migration.tmp"));
     286    for (Transfer t: transfers)
     287    {
     288      log.info("Store to resume point: " + t);
     289      t.writeRestorePointData(oos);
     290    }
     291    oos.close();
     292  }
     293 
     294  private void removeResumePoint()
     295  {
     296    log.info("Removing resume point");
     297    java.io.File resumePoint = new java.io.File("migration.tmp");
     298    resumePoint.delete();
     299  }
     300 
    212301  private void startTransfer(Transfer transfer, String property)
    213302  {
     
    474563    System.err.println("-----------------------------------------------------------------------");
    475564
     565    java.io.File restorePoint = new java.io.File("migration.tmp");
     566    boolean resumeMigration = false;
     567
    476568    if (args == null || args.length == 0 || !"disclaimer_understood".equals(args[0]))
    477569    {
     
    479571      System.exit(0);
    480572    }
     573    else if (restorePoint.exists())
     574    {
     575      System.err.println("FOUND DATA FROM A PREVIOUSLY INTERRUPTED MIGRATION.");
     576      System.err.println("\n   Press ENTER to resume the migration\nor\n   Ctrl-C to abort\n");
     577      System.err.println("To start a new migration delete the './migration.tmp' file");
     578      System.err.println("before starting the migration program.");
     579      waitForEnter();
     580      resumeMigration = true;
     581    }
    481582    else
    482583    {
     
    485586    }
    486587   
    487     System.err.println("Starting migration...");
     588    System.err.println(resumeMigration ? "Resuming migration" : "Starting migration...");
    488589    Application.start(false);
    489590    Migrater migrater = new Migrater();
     
    493594    migrater.prepareStatementsFromFile();
    494595    log.info("Migration initialisation DONE");
    495     migrater.run();
     596    if (resumeMigration)
     597    {
     598      migrater.resume();
     599    }
     600    else
     601    {
     602      migrater.run();
     603    }
    496604  }
    497605 
     
    506614    {}
    507615  }
    508  
    509  
     616   
    510617}
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/QuotaTransfer.java

    r3683 r3709  
    9595      quota.setName(formatBytes(totalQuota));
    9696      dc.saveItem(quota);
     97      quotas.put(totalQuota, quota);
    9798    }
    9899    else
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayDataTransfer.java

    r3683 r3709  
    4747
    4848public class RawBioAssayDataTransfer
    49   extends CommonItemTransfer
     49  extends Transfer
    5050{
    5151  /**
     
    7777  @param groupT A finished {@link GroupTransfer}
    7878   */
    79   RawBioAssayDataTransfer(UserTransfer userT, GroupTransfer groupT,
    80       RawBioAssayTransfer rbaT, ArrayTypeTransfer arrayTypeT,
    81       ReporterTransfer reporterT)
    82   {
    83     super(userT, groupT);
     79  RawBioAssayDataTransfer(RawBioAssayTransfer rbaT, ReporterTransfer reporterT)
     80  {
     81    super();
    8482    log.debug("Checking that necessary transfers have been done before proceeding with RawBioAssayData");
    85     if (rbaT != null && rbaT.isDone() && arrayTypeT != null
    86         && arrayTypeT.isDone() && reporterT != null
    87         && reporterT.isDone())
     83    if (rbaT != null && rbaT.isDone() &&
     84      reporterT != null && reporterT.isDone())
    8885    {
    8986      this.rbaT = rbaT;
     
    9390    {
    9491      if (!rbaT.isDone()) log.debug("Dependency: RawBioAssays haven't been transfered");
    95       if (!arrayTypeT.isDone()) log.debug("Dependency: ArrayTypes haven't been transfered");
    9692      if (!reporterT.isDone()) log.debug("Dependency: Reporters haven't been transfered");
    9793      dependencyFailure();
     
    133129        log.debug("RawBioAssay '" + rawBioAssayId + "' " + (hasArrayDesign ? "has " : "no ") + "Array Design");
    134130      }
    135       transferRawBioAssayData(rawBioAssay, rawBioAssayId, hasArrayDesign);
     131      if (!rawBioAssay.hasData())
     132      {
     133        transferRawBioAssayData(rawBioAssay, rawBioAssayId, hasArrayDesign);
     134      }
     135      else
     136      {
     137        log.info("Skipping since it already has data");
     138        progress.increase(rawBioAssay.getSpots());
     139      }
    136140      progress.decrease(); // We do not want to count the RawBioAssays
    137141      if (log.isDebugEnabled())
     
    263267      if (log.isInfoEnabled())
    264268      {
    265         log.info("Transfered " + rowCount + " rows of RawData for raw bioassay" + rawBioAssay.getName());
     269        log.info("Transfered " + rowCount + " rows of RawData for raw bioassay: " + rawBioAssay.getName());
    266270      }
    267271    }
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/RawBioAssayTransfer.java

    r3683 r3709  
    6363  private SampleTransfer sampleT = null;
    6464
     65  RawBioAssayTransfer()
     66  {}
     67 
    6568  /**
    6669  Constructs a new RawBioAssayTransfer.
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/ReporterTransfer.java

    r3683 r3709  
    9090 
    9191  /**
     92    Flag that indicates if we should run in RESUME mode
     93    (ie. NOT create new reporter items, only map ID:s)
     94  */
     95  private boolean RESUME = false;
     96 
     97  /**
    9298  Set string and text properties, order must match the select statement
    9399  except for the id and reporterId in the beginning Extended properties are
     
    102108  Reporters are independant from other transfers.
    103109   */
    104   ReporterTransfer()
    105   {}
     110  ReporterTransfer(boolean resume)
     111  {
     112    RESUME = resume;
     113  }
    106114
    107115  public void start()
    108116  {
    109     if (!QUICK_TRANSFER) log.info("Transferring reporters...");
    110     else log.info("Transferring reporters with minimum properties...");
     117    if (RESUME) QUICK_TRANSFER = true;
     118    if (RESUME)
     119    {
     120      log.info("Loading reporters...");
     121    }
     122    else if (!QUICK_TRANSFER)
     123    {
     124      log.info("Transferring reporters...");
     125    }
     126    else
     127    {
     128      log.info("Transferring reporters with minimum properties...");
     129    }
    111130   
    112131    int count = getInt("countReporters");
     
    199218        rowCount++;
    200219        ReporterData item = createDataItem(rs);
    201         if (item != null) batcher.insert(item);
     220        if (item != null && !RESUME) batcher.insert(item);
    202221        progress.increase();
    203222      }
     
    319338      }
    320339      rd = Reporter.getNew(externalId);
     340      String name = rs.getString(3);
     341      if (name == null || "".equals(name)) name = externalId;
     342      rd.setName(name);
     343      rd.setSymbol(rs.getString(4));
    321344      if (!QUICK_TRANSFER)
    322345      {
    323         String name = rs.getString(3);
    324         if (name == null || "".equals(name)) name = externalId;
    325         rd.setName(name);
    326         rd.setSymbol(rs.getString(4));
    327346        int columnIndex = 5;
    328347        for (String property : stringProperties)
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/SharedItemTransfer.java

    r3679 r3709  
    8383  private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.SharedItemTransfer");
    8484
     85  SharedItemTransfer()
     86  {}
     87 
    8588  SharedItemTransfer(UserTransfer userT, GroupTransfer groupT)
    8689  {
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/Transfer.java

    r3683 r3709  
    3232import net.sf.basedb.core.ItemAlreadyExistsException;
    3333import net.sf.basedb.core.SessionControl;
     34
     35import java.io.IOException;
     36import java.io.ObjectInputStream;
     37import java.io.ObjectOutputStream;
    3438import java.sql.Connection;
    3539import java.sql.PreparedStatement;
     
    613617    return Integer.parseInt(value);
    614618  }
     619 
     620  /**
     621    Store ID map and some boolean flags
     622  */
     623  protected void writeRestorePointData(ObjectOutputStream out)
     624    throws IOException
     625  {
     626    out.writeBoolean(dependencyFailed);
     627    out.writeBoolean(done);
     628    out.writeBoolean(idMap != null);
     629    if (idMap != null) out.writeObject(idMap);
     630  }
     631 
     632  /**
     633    Store ID map and some boolean flags
     634  */
     635  @SuppressWarnings("unchecked")
     636  protected void readRestorePointData(ObjectInputStream in)
     637    throws IOException, ClassNotFoundException
     638  {
     639    dependencyFailed = in.readBoolean();
     640    done = in.readBoolean();
     641    if (in.readBoolean())
     642    {
     643      idMap = (Map<Integer, Integer>)in.readObject();
     644    }
     645  }
     646
    615647}
  • branches/2.4-stable/src/clients/migrate/net/sf/basedb/clients/migrate/UserTransfer.java

    r3683 r3709  
    3636import net.sf.basedb.core.SystemItems;
    3737import net.sf.basedb.core.User;
     38
     39import java.io.IOException;
     40import java.io.ObjectInputStream;
     41import java.io.ObjectOutputStream;
    3842import java.sql.PreparedStatement;
    3943import java.sql.ResultSet;
     
    9094   */
    9195  private static final org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger("net.sf.basedb.clients.migrate.UserTransfer");
     96 
     97  UserTransfer(QuotaTransfer quotaT)
     98  {
     99    this.quotaT = quotaT;
     100  }
    92101 
    93102  /**
     
    323332  }
    324333 
     334  /**
     335    Store quota map
     336  */
     337  protected void writeRestorePointData(ObjectOutputStream out)
     338    throws IOException
     339  {
     340    super.writeRestorePointData(out);
     341    out.writeObject(base1FileQuota);
     342  }
     343 
     344  /**
     345    Store ID map and some boolean flags
     346  */
     347  @SuppressWarnings("unchecked")
     348  protected void readRestorePointData(ObjectInputStream in)
     349    throws IOException, ClassNotFoundException
     350  {
     351    super.readRestorePointData(in);
     352    base1FileQuota = (Map<Integer, Long>)in.readObject();
     353  }
     354 
    325355  public User getUserById(DbControl dc, int base1Id)
    326356  {
Note: See TracChangeset for help on using the changeset viewer.