View Javadoc
1   // ******************************************************************************
2   //
3   // Title:       Force Field X.
4   // Description: Force Field X - Software for Molecular Biophysics.
5   // Copyright:   Copyright (c) Michael J. Schnieders 2001-2025.
6   //
7   // This file is part of Force Field X.
8   //
9   // Force Field X is free software; you can redistribute it and/or modify it
10  // under the terms of the GNU General Public License version 3 as published by
11  // the Free Software Foundation.
12  //
13  // Force Field X is distributed in the hope that it will be useful, but WITHOUT
14  // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15  // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
16  // details.
17  //
18  // You should have received a copy of the GNU General Public License along with
19  // Force Field X; if not, write to the Free Software Foundation, Inc., 59 Temple
20  // Place, Suite 330, Boston, MA 02111-1307 USA
21  //
22  // Linking this library statically or dynamically with other modules is making a
23  // combined work based on this library. Thus, the terms and conditions of the
24  // GNU General Public License cover the whole combination.
25  //
26  // As a special exception, the copyright holders of this library give you
27  // permission to link this library with independent modules to produce an
28  // executable, regardless of the license terms of these independent modules, and
29  // to copy and distribute the resulting executable under terms of your choice,
30  // provided that you also meet, for each linked independent module, the terms
31  // and conditions of the license of that module. An independent module is a
32  // module which is not derived from or based on this library. If you modify this
33  // library, you may extend this exception to your version of the library, but
34  // you are not obligated to do so. If you do not wish to do so, delete this
35  // exception statement from your version.
36  //
37  // ******************************************************************************
38  package ffx.algorithms.dynamics;
39  
40  import static ffx.utilities.Constants.KCAL_TO_GRAM_ANG2_PER_PS2;
41  import static ffx.utilities.Constants.kB;
42  import static org.apache.commons.math3.util.FastMath.exp;
43  
44  import edu.rit.mp.DoubleBuf;
45  import edu.rit.mp.IntegerBuf;
46  import edu.rit.mp.buf.IntegerMatrixBuf_1;
47  import edu.rit.pj.Comm;
48  import ffx.algorithms.Terminatable;
49  import ffx.numerics.Potential;
50  import ffx.potential.extended.ExtendedSystem;
51  import java.io.BufferedReader;
52  import java.io.File;
53  import java.io.FileReader;
54  import java.io.IOException;
55  import java.nio.file.Files;
56  import java.nio.file.StandardCopyOption;
57  import java.util.ArrayList;
58  import java.util.Arrays;
59  import java.util.List;
60  import java.util.Random;
61  import java.util.logging.Level;
62  import java.util.logging.Logger;
63  
64  import ffx.potential.parameters.TitrationUtils;
65  import jline.internal.Nullable;
66  import org.apache.commons.io.FilenameUtils;
67  
68  /**
69   * The PhReplicaExchange implements pH replica exchange. Adapted from "ReplicaExchange.java" by
70   * Timothy D. Fenn and Michael J. Schnieders
71   *
72   * @author Matthew Speranza and Andrew Thiel
73   * @since 1.0
74   */
75  public class PhReplicaExchange implements Terminatable {
76  
77    private static final Logger logger = Logger.getLogger(PhReplicaExchange.class.getName());
78    private final MolecularDynamics replica;
79    private final MolecularDynamicsOpenMM openMM;
80    private final Potential potential;
81    private final ExtendedSystem extendedSystem;
82    private final Random random;
83    /** Parallel Java world communicator. */
84    private final Comm world;
85    private File dyn, esv, esvBackup, dynBackup;
86    private ArrayList<Double> readPhScale;
87    /**
88     * Each parameter array is wrapped inside a Parallel Java DoubleBuf for the All-Gather
89     * communication calls.
90     */
91    private final DoubleBuf[] parametersBuf;
92    private final DoubleBuf myParametersBuf;
93    /**
94     * The parameters array stores communicated parameters for each process (i.e. each RepEx system).
95     * Currently, the array is of size [number of Processes][2].
96     */
97    private final double[][] parameters;
98    private final int[][][] parametersHis;
99    private final IntegerBuf[] parametersHisBuf;
100   private final int[] pH2Rank, rank2Ph, pHAcceptedCount, pHTrialCount;
101   /** Rank of this process. */
102   private final int rank, nReplicas;
103   private boolean done = false, restart, terminate = false, backupNeeded = false;
104   private final double[] myParameters, pHScale;
105   private double[] x;
106   private final double pH, temp;
107   private int restartStep;
108 
109   /**
110    * pHReplicaExchange constructor.
111    *
112    * @param molecularDynamics a {@link MolecularDynamics} object.
113    * @param pH pH = pKa will be changed from this initial value
114    * @param extendedSystem extended system attached to this process
115    * @param pHLadder range of pH's that replicas are created for
116    * @param temp temperature of replica
117    */
118   public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
119                            double[] pHLadder, double temp, ExtendedSystem extendedSystem, double[] x, int worldSize) throws Exception {
120     this(molecularDynamics, structureFile, pH, pHLadder, temp, extendedSystem, x, null, null, worldSize);
121   }
122 
123   /**
124    * OpenMM cycled pHReplicaExchange constructor.
125    *
126    * @param molecularDynamics a {@link MolecularDynamics} object.
127    * @param pH pH = pKa will be changed from this initial value
128    * @param extendedSystem extended system attached to this process
129    * @param pHLadder range of pH's that replicas are created for
130    * @param temp temperature of replica
131    * @param x array of coordinates
132    * @param molecularDynamicsOpenMM for running config steps on GPU
133    */
134   public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
135                            double[] pHLadder, double temp, ExtendedSystem extendedSystem, @Nullable double[] x,
136                            @Nullable MolecularDynamicsOpenMM molecularDynamicsOpenMM, @Nullable Potential potential,
137                            int worldSize) throws Exception {
138     this.replica = molecularDynamics;
139     this.temp = temp;
140     this.extendedSystem = extendedSystem;
141     this.pH = pH;
142     this.pHScale = pHLadder;
143     this.x = x;
144     this.openMM = molecularDynamicsOpenMM;
145     this.potential = potential;
146 
147     // Set up the Replica Exchange communication variables for Parallel Java communication between nodes.
148     world = Comm.world();
149 
150     // Number of processes is equal to the number of replicas.
151     int numProc = worldSize;
152     rank = world.rank();
153 
154     nReplicas = numProc;
155     pH2Rank = new int[nReplicas];
156     rank2Ph = new int[nReplicas];
157     pHAcceptedCount = new int[nReplicas];
158     pHTrialCount = new int[nReplicas];
159 
160     // Init map
161     for (int i = 0; i < nReplicas; i++) {
162       rank2Ph[i] = i;
163       pH2Rank[i] = i;
164     }
165     extendedSystem.setConstantPh(pHScale[rank]);
166 
167     random = new Random();
168     random.setSeed(0);
169 
170     // Create arrays to store the parameters of all processes.
171     parameters = new double[nReplicas][pHLadder.length + 1];
172     parametersHis = new int[nReplicas][extendedSystem.getTitratingResidueList().size()][100];
173     parametersBuf = new DoubleBuf[nReplicas];
174     parametersHisBuf = new IntegerMatrixBuf_1[nReplicas];
175     for (int i = 0; i < nReplicas; i++) {
176       parametersBuf[i] = DoubleBuf.buffer(parameters[i]);
177       parametersHisBuf[i] = IntegerMatrixBuf_1.buffer(parametersHis[i]);
178     }
179 
180     // A convenience reference to the parameters of this process are updated during communication calls.
181     myParameters = parameters[rank];
182     myParametersBuf = parametersBuf[rank];
183 
184     // Set the ESV that this rank will share has correct values after a restart
185     if (manageFilesAndRestart(structureFile)) {
186       logger.info(" Startup Successful! ");
187       extendedSystem.getESVHistogram(parametersHis[rank]);
188     } else {
189       throw new Exception("Failed to restart Replica Exchange.");
190     }
191   }
192 
193   /**
194    * This deals with anything having to do with restarts and backup files. Is identical between CPU
195    * and GPU.
196    * <p>
197    * Restart Rules:
198    * <p>
199    * 1. Checks for existence of primary restart files in all directories.
200    * <p>
201    * 2. Check for even sums of esv samples.
202    * <p>
203    * 3. Check for unique pH values in esv files.
204    * <p>
205    * 4. Fall back and repeat 1-3 with backup files if primary backup failed.
206    * <p>
207    * 5. Return true if all checks pass, false if any fail in both primary and backup.
208    * <p>
209    * Treats as fresh start if no restart files are found or if the primary are found, fail,
210    * and backups are not found.
211    *
212    * @param structureFile pdb file
213    * @return whether the restart was successful or not
214    */
215   public boolean manageFilesAndRestart(File structureFile) {
216     // My directory info
217     File parent = structureFile.getParentFile();
218     File rankDir = new File(parent + File.separator + rank);
219     restart = !rankDir.mkdir();
220 
221     // My files
222     esv = new File(
223             rankDir.getPath() + File.separator + FilenameUtils.removeExtension(structureFile.getName())
224                     + ".esv");
225     dyn = new File(
226             rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
227                     + ".dyn");
228     esvBackup = new File(FilenameUtils.removeExtension(esv.getAbsolutePath()) + "_backup.esv");
229     dynBackup = new File(FilenameUtils.removeExtension(dyn.getAbsolutePath()) + "_backup.dyn");
230 
231     // Set restart files - does not do any reading yet
232     extendedSystem.setRestartFile(esv);
233     replica.setFallbackDynFile(dyn);
234     replica.setArchiveFiles(new File[]{new File(
235             rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
236                     + ".arc")});
237     if (openMM != null) {
238       openMM.setAutomaticWriteouts(false);
239     }
240 
241     // Build restart
242     restartStep = 0;
243     if (restart) {
244       logger.info(" Attempting to restart. ");
245       readPhScale = new ArrayList<>();
246 
247       // Read normal restarts
248       backupNeeded = false;
249       if (checkForRestartFiles(parent, esv.getName()) && checkForRestartFiles(parent,
250               dyn.getName())) {
251         for (int i = 0; i < nReplicas; i++) {
252           File checkThisESV = new File(
253                   parent.getAbsolutePath() + File.separator + i + File.separator + esv.getName());
254           backupNeeded = !readESV(checkThisESV, i);
255           if (backupNeeded) {
256             logger.warning(" Searching backup esv files.");
257             break;
258           }
259         }
260         if (!backupNeeded) {
261           logger.info(" Reading in from: " + esv);
262           extendedSystem.readESVInfoFrom(esv);
263         }
264       } else {
265         logger.info(" Directories do not contain all of the correct restart files.");
266         backupNeeded = true;
267       }
268 
269       // Read backup restarts
270       if (backupNeeded && checkForRestartFiles(parent, esvBackup.getName()) && checkForRestartFiles(
271               parent, dynBackup.getName())) {
272         for (int i = 0; i < nReplicas; i++) {
273           File checkThisESV = new File(
274                   parent.getAbsolutePath() + File.separator + i + File.separator + esvBackup.getName());
275           if (!readESV(checkThisESV, i)) {
276             logger.info(" Restart files & Backups are messed up.");
277             return false;
278           }
279         }
280         logger.info(" Reading in from: " + esvBackup);
281         extendedSystem.readESVInfoFrom(esvBackup);
282       } else if (backupNeeded) {
283         logger.info(" Directories do not contain all of the correct backup restart files.");
284         logger.info(" No backup files found, treating as a fresh start.");
285         restart = false;
286         return true;
287       }
288     } else {
289       logger.info(" Not a restart.");
290       return true;
291     }
292 
293     logger.info(" Rank " + rank + " staring at pH " + pHScale[rank2Ph[rank]]);
294     extendedSystem.setConstantPh(pHScale[rank2Ph[rank]]);
295     logger.info(" Rank " + rank + " starting hist:");
296     extendedSystem.writeLambdaHistogram(true);
297 
298     return true;
299   }
300 
301   /**
302    * Reads an esv file to find the number of steps logged and if the pH matches the others in
303    * readPhScale. (N.B. - This only checks the first esv in the system for uneven sums, since it is
304    * assumed that all counts will be messed up if the first one is.)
305    *
306    * @param file to read
307    * @return if the esv file of this name at this rank works with the others, or if i=0, then it sets
308    *     the standards
309    */
310   private boolean readESV(File file, int i) {
311     try (BufferedReader br = new BufferedReader(new FileReader(file))) {
312       String data = br.readLine();
313       while (data != null) {
314         List<String> tokens = Arrays.asList(data.split(" +"));
315         if (tokens.contains("pH:")) {
316           double pHOfRankI = Double.parseDouble(tokens.get(tokens.indexOf("pH:") + 1));
317 
318           // Set up map
319           for (int j = 0; j < pHScale.length; j++) {
320             if (Math.abs(pHScale[j] / pHOfRankI - 1) < 0.00001) {
321               rank2Ph[i] = j;
322               pH2Rank[j] = i;
323             }
324           }
325 
326           // Check pH list for duplicates
327           if (readPhScale.stream().anyMatch(d -> (Math.abs(d / pHOfRankI - 1) < 0.00001))) {
328             logger.warning(" Duplicate pH value found in file: " + file.getAbsolutePath());
329             readPhScale.clear();
330             restartStep = 0;
331             return false;
332           } else {
333             readPhScale.add(pHOfRankI);
334           }
335 
336           // Find restart steps
337           br.readLine();
338           br.readLine();
339           int sum = 0;
340           for (int j = 0; j < 10; j++) { // 10 titr windows
341             data = br.readLine().trim();
342             tokens = List.of(data.split(" +"));
343             for (int k = 0; k < 10; k++) { // 10 tautomer windows
344               sum += Integer.parseInt(tokens.get(k + 1));
345             }
346           }
347 
348           if (restartStep == 0) {
349             restartStep = sum;
350           } else if (restartStep != sum) {
351             logger.warning(" Restart received uneven sums from esv file: " + file.getAbsolutePath());
352             logger.info(" Restart Step: " + restartStep + "    Found: " + sum);
353             restartStep = 0;
354             readPhScale.clear();
355             return false;
356           }
357           break;
358         }
359         data = br.readLine();
360       }
361       return true;
362     } catch (IOException | IndexOutOfBoundsException e) {
363       logger.warning("Failed to read file: " + file.getAbsolutePath());
364       return false;
365     }
366   }
367 
368   /**
369    * Looks through directories to see if they all contain the right file
370    *
371    * @return whether they all contain the correct file
372    */
373   private boolean checkForRestartFiles(File parent, String fileNameWithExtension) {
374     File searchFile;
375     for (int i = 0; i < nReplicas; i++) {
376       searchFile = new File(
377               parent.getAbsolutePath() + File.separator + i + File.separator + fileNameWithExtension);
378       if (!searchFile.exists()) {
379         return false;
380       }
381     }
382     return true;
383   }
384 
385   /**
386    * Sample. Restart file write-outs are entirely handled by this method based on how many steps are per cycle.
387    * User command input is ignored.
388    *
389    * @param cycles The number of cycles to run.
390    * @param nSteps The number of steps per cycle.
391    * @param timeStep The time step.
392    * @param printInterval The print interval.
393    */
394   public void sample(int cycles, long nSteps, double timeStep, double printInterval,
395                      double trajInterval, int initTitrDynamics) {
396     sample(cycles, nSteps, 0, timeStep, printInterval, trajInterval, initTitrDynamics);
397   }
398 
399   public void sample(int cycles, long titrSteps, long confSteps, double timeStep,
400                      double printInterval, double trajInterval, int initDynamics) {
401     done = false;
402     terminate = false;
403     replica.setRestartFrequency(cycles * (titrSteps + confSteps) * replica.dt + 100); // Full control over restarts handled by this class
404     extendedSystem.reGuessLambdas();
405     replica.setCoordinates(x);
406     int startCycle = 0;
407     if (initDynamics > 0 && !restart) {
408       extendedSystem.reGuessLambdas();
409       extendedSystem.setFixedTitrationState(true);
410       extendedSystem.setFixedTautomerState(true);
411 
412       logger.info(" ");
413       logger.info(" ------------------Start of Equilibration Dynamics------------------\n");
414       logger.info(" ");
415 
416       if (openMM == null) {
417         replica.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
418       } else {
419         x = replica.getCoordinates();
420         potential.energy(x);
421         openMM.setCoordinates(x);
422 
423         openMM.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
424 
425         x = openMM.getCoordinates();
426         replica.setCoordinates(x);
427       }
428 
429       extendedSystem.setFixedTitrationState(false);
430       extendedSystem.setFixedTautomerState(false);
431       logger.info(extendedSystem.getLambdaList());
432       extendedSystem.writeLambdaHistogram(true);
433       extendedSystem.copyESVHistogramTo(parametersHis[rank]); // Copy the ESV hist to be empty
434 
435       logger.info(" ");
436       logger.info(" ------------------End of Equilibration Dynamics------------------\n");
437       logger.info(" ");
438     } else if (restart) {
439       logger.info(" Omitting initialization steps because this is a restart.");
440       startCycle = (int) (restartStep / titrSteps) + 1;
441       logger.info(" Restarting pH-REX at cycle " + startCycle + " of " + cycles);
442     }
443 
444     for (int i = startCycle; i < cycles; i++) {
445       // Check for termination request.
446       if (terminate) {
447         done = true;
448         break;
449       }
450 
451       if (openMM != null) {
452         if (confSteps < 3) {
453           logger.severe(" Increase number of steps per cycle.");
454         }
455         dynamicsOpenMM(titrSteps, confSteps, timeStep, printInterval, trajInterval);
456       } else {
457         dynamics(titrSteps, timeStep, printInterval, trajInterval);
458       }
459       // Set backups in case job is killed at bad time
460       if (i == 0 || backupNeeded) {
461         replica.writeRestart();
462       }
463       copyToBackups();
464       replica.writeRestart();
465 
466       if (i % 100 == 0) {
467         extendedSystem.writeLambdaHistogram(true);
468       }
469 
470       try {
471         logger.info("\n Predicting pKa...");
472         logger.info(pkaPred(parametersHis));
473       } catch (Exception e) {
474         logger.info(e.getMessage());
475         logger.warning("Failed to predict pKa. Simulation may be too early on.");
476       }
477 
478       logger.info(" ");
479       logger.info(String.format(" ------------------Exchange Cycle %d------------------\n", i + 1));
480 
481       if (i != cycles - 1) {
482         exchange();
483       }
484       logger.info(" ");
485       logger.info(" Setting rank " + rank + " esv to pH " + pHScale[rank2Ph[rank]]);
486       logger.info(" ");
487     }
488   }
489 
490   private String pkaPred(int[][][] parametersHis) {
491     // parametersHis shape explanation - [rank (w/ some pH)][residue][histogram --> len = 100]
492     // Calculate collapsed fractions of 10x10 histograms
493     double[][][] collapsedSum = new double[parametersHis.length][parametersHis[0].length][10];
494     double[][] collapsedRatio = new double[parametersHis.length][parametersHis[0].length];
495     for(int i = 0; i < parametersHis.length; i++) // pH's
496     {
497       for(int j = 0; j < parametersHis[0].length; j++) // residues
498       {
499         for(int k = 0; k < 10; k++) // 10 titration states
500         {
501           double sum = 0;
502           for(int l = 0; l < 10; l++) // 10 tautomer states
503           {
504             sum += parametersHis[i][j][k*10 + l];
505           }
506           collapsedSum[i][j][k] = sum;
507         }
508         // Calculate ratio over 0 and 9
509         double ratio = collapsedSum[i][j][0] / (collapsedSum[i][j][0] + collapsedSum[i][j][9]);
510         collapsedRatio[i][j] = ratio;
511       }
512     }
513 
514     // Calculate hill coeff and pKa's of each residue and print
515     StringBuilder output = new StringBuilder();
516     double[] n = new double[parametersHis[0].length];
517     double[] pka = new double[parametersHis[0].length];
518     double[][] residueRatios = new double[parametersHis[0].length][parametersHis.length];
519     for(int i = 0; i < parametersHis[0].length; i++) // residues
520     {
521       // Create array of fractions for this residue
522       for(int j = 0; j < parametersHis.length; j++) // pH's
523       {
524         residueRatios[i][j] = collapsedRatio[j][i];
525       }
526       //Arrays.sort(residueRatios[i]);
527 
528       // L-BFGS minimization of the Henderson-Hasselbalch equation to find the best fit hill coeff and pKa
529       double[] temp = TitrationUtils.predictHillCoeffandPka(pHScale, residueRatios[i]);
530       n[i] = temp[0];
531       pka[i] = temp[1];
532 
533       // Print results for this residue
534       String residueName = extendedSystem.getTitratingResidueList().get(i).toString();
535       output.append(" Residue: ").append(residueName).append("\n");
536       output.append(" Fractions (Dep / (Dep + Pro)): ").append(Arrays.toString(residueRatios[i])).append("\n");
537       output.append(" pH window: ").append(Arrays.toString(pHScale)).append("\n");
538       output.append(" n: ").append(n[i]).append("\n");
539       output.append(" pKa: ").append(pka[i]).append("\n");
540       output.append("\n");
541     }
542 
543     return output.toString();
544   }
545 
546   /**
547    * Sets an even pH ladder based on the pH gap.
548    */
549   public static double[] setEvenSpacePhLadder(double pHGap, double pH, int nReplicas) {
550     double range = nReplicas * pHGap;
551     double pHMin = pH - range / 2;
552 
553     if (nReplicas % 2 != 0) {
554       pHMin += pHGap / 2; // Center range if odd num windows
555     }
556 
557     double[] pHScale = new double[nReplicas];
558     for (int i = 0; i < nReplicas; i++) {
559       pHScale[i] = pHMin + i * pHGap;
560     }
561 
562     return pHScale;
563   }
564 
565   /**
566    * {@inheritDoc}
567    *
568    * <p>This should be implemented as a blocking interrupt; when the method returns the <code>
569    * Terminatable</code> algorithm has reached a clean termination point. For example, between
570    * minimize or molecular dynamics steps.
571    */
572   @Override
573   public void terminate() {
574     terminate = true;
575     while (!done) {
576       synchronized (this) {
577         try {
578           wait(1);
579         } catch (InterruptedException e) {
580           logger.log(Level.WARNING, "Exception terminating replica exchange.\n", e);
581         }
582       }
583     }
584   }
585 
586   /**
587    * Evaluate whether to exchange.
588    *
589    * @param pH The pH to have as the replica target.
590    */
591   private void compareTwo(int pH) {
592     // Ranks for pH A and B
593     int rankA;
594     int rankB;
595 
596     // Load pH, beta and energy for each rank.
597     double pHA;
598     double pHB;
599 
600     double beta;
601     double acidostatA;
602     double acidostatB;
603     double acidostatAatB;
604     double acidostatBatA;
605 
606     rankA = pH2Rank[pH];
607     rankB = pH2Rank[pH + 1];
608     int acidAIndex = rank2Ph[rankA] + 1;
609     int acidBIndex = rank2Ph[rankB] + 1;
610 
611     pHA = parameters[rankA][0];
612     pHB = parameters[rankB][0];
613 
614     beta = KCAL_TO_GRAM_ANG2_PER_PS2 / (temp * kB);
615 
616     acidostatA = parameters[rankA][acidAIndex];
617     acidostatB = parameters[rankB][acidBIndex];
618 
619     acidostatAatB = parameters[rankA][acidBIndex]; // acidostat of rankA evaluated at the pH of rankB
620     acidostatBatA = parameters[rankB][acidAIndex];
621 
622     logger.info(" ");
623     logger.info(
624             " From rank " + rank + ": Comparing ranks " + rankA + " (pH = " + pHA + ") & " + rankB
625                     + " (pH = " + pHB + ")");
626 
627     // Compute the change in energy over kT (E/kT) for the Metropolis criteria.
628     double deltaE = beta * ((acidostatAatB + acidostatBatA) - (acidostatA + acidostatB));
629     logger.info("pHA = " + pHA + " " + acidostatA + " " + acidostatAatB);
630     logger.info("pHB = " + pHB + " " + acidostatB + " " + acidostatBatA);
631     logger.info("exp(" + beta + " * ((" + acidostatAatB + " + " + acidostatBatA + ") - (" + acidostatA + " + " + acidostatB + ")))");
632     logger.info(" DeltaE: " + deltaE);
633 
634     //Count the number of trials for each temp
635     pHTrialCount[pH]++;
636 
637     // If the Metropolis criteria is satisfied, do the switch.
638     if (deltaE < 0.0 || random.nextDouble() < exp(-deltaE)) {
639       int[][] tempHis = new int[extendedSystem.getTitratingResidueList().size()][100];
640       for (int i = 0; i < extendedSystem.getTitratingResidueList().size(); i++) {
641         System.arraycopy(parametersHis[rankA][i], 0, tempHis[i], 0, tempHis[i].length);
642         System.arraycopy(parametersHis[rankB][i], 0, parametersHis[rankA][i], 0,
643                 parametersHis[rankA][i].length);
644         System.arraycopy(tempHis[i], 0, parametersHis[rankB][i], 0, parametersHis[rankB][i].length);
645       }
646       parameters[rankA][0] = pHB;
647       parameters[rankB][0] = pHA;
648 
649       //Log the new parameters
650       logger.info(" pH After swap " + parameters[rankA][0] + " rankA " + rankA + " acidostat: " + Arrays.toString(parameters[rankA]));
651       logger.info(" pH After swap " + parameters[rankB][0] + " rankB " + rankB + " acidostat: " + Arrays.toString(parameters[rankB]));
652 
653       // Map pH to process ranks.
654       pH2Rank[pH] = rankB;
655       pH2Rank[pH + 1] = rankA;
656 
657       // Map ranks to pH.
658       rank2Ph[rankA] = pH + 1;
659       rank2Ph[rankB] = pH;
660 
661       pHAcceptedCount[pH]++;
662     }
663   }
664 
665   /** All processes complete the exchanges identically given the same Random number seed. */
666   private void exchange() {
667     // Loop over top and bottom parts of pH scale
668     for (int pH = 0; pH < nReplicas - 1; pH++) {
669       compareTwo(pH);
670     }
671 
672     logger.info(" ");
673     // Print Exchange Info
674     for (int i = 0; i < pHScale.length - 1; i++) {
675       double pHAcceptance = pHAcceptedCount[i] * 100.0 / (pHTrialCount[i]);
676       logger.info(
677               " Acceptance for pH " + pHScale[i] + " to be exchanged with pH " + pHScale[i + 1] + ": "
678                       + pHAcceptance);
679     }
680     logger.info(" ");
681   }
682 
683   /**
684    * Blocking dynamic steps: when this method returns each replica has completed the requested number
685    * of steps. Both OpenMM and CPU implementations exist
686    *
687    * @param titrSteps the number of time steps on CPU.
688    * @param confSteps the number of time steps on GPU.
689    * @param timeStep the time step.
690    * @param printInterval the number of steps between logging updates.
691    * @param saveInterval the number of steps between saving snapshots.
692    */
693   private void dynamicsOpenMM(final long titrSteps, final long confSteps, final double timeStep,
694                               final double printInterval, final double saveInterval) {
695 
696     int i = rank2Ph[rank];
697     // Update this ranks' parameter array to be consistent with the dynamics.
698     myParameters[0] = pHScale[i];
699     extendedSystem.setConstantPh(myParameters[0]);
700     extendedSystem.copyESVHistogramTo(parametersHis[rank]);
701 
702     // Start this processes MolecularDynamics instance sampling.
703     boolean initVelocities = true;
704 
705     x = replica.getCoordinates();
706     potential.energy(x);
707     openMM.setCoordinates(x);
708     
709     openMM.dynamic(confSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
710     x = openMM.getCoordinates();
711     replica.setCoordinates(x);
712 
713     double forceWriteInterval = titrSteps * .001;
714     replica.dynamic(titrSteps, timeStep, printInterval, forceWriteInterval, temp, initVelocities, dyn);
715     reEvaulateAcidostats();
716 
717     extendedSystem.getESVHistogram(parametersHis[rank]);
718 
719     // Gather all parameters from the other processes.
720     try {
721       world.allGather(myParametersBuf, parametersBuf);
722       world.allGather(parametersHisBuf[rank], parametersHisBuf);
723     } catch (IOException ex) {
724       String message = " Replica Exchange allGather failed.";
725       logger.log(Level.SEVERE, message, ex);
726     }
727   }
728 
729 
730   private void dynamics(long nSteps, double timeStep, double printInterval, double saveInterval) {
731     int i = rank2Ph[rank];
732 
733     // Update this ranks' parameter array to be consistent with the dynamics.
734     myParameters[0] = pHScale[i];
735     extendedSystem.setConstantPh(myParameters[0]);
736 
737     extendedSystem.copyESVHistogramTo(parametersHis[rank]);
738 
739     // Start this processes MolecularDynamics instance sampling.
740     boolean initVelocities = true;
741 
742     replica.dynamic(nSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
743 
744     reEvaulateAcidostats();
745 
746     extendedSystem.getESVHistogram(parametersHis[rank]);
747 
748     // Gather all parameters from the other processes.
749     try {
750       world.allGather(myParametersBuf, parametersBuf);
751       world.allGather(parametersHisBuf[rank], parametersHisBuf);
752     } catch (IOException ex) {
753       String message = " Replica Exchange allGather failed.";
754       logger.log(Level.SEVERE, message, ex);
755     }
756   }
757 
758   private void reEvaulateAcidostats() {
759     for(int j = 0; j < pHScale.length; j++){
760       extendedSystem.setConstantPh(pHScale[j]); // A at A+gap(B)
761       myParameters[1+j] = extendedSystem.getBiasEnergy();
762     }
763 
764     extendedSystem.setConstantPh(myParameters[0]);
765   }
766 
767   private void copyToBackups() {
768     try {
769       Files.move(esv.toPath(), esv.toPath().resolveSibling(esvBackup.getName()),
770               StandardCopyOption.REPLACE_EXISTING);
771     } catch (IOException e) {
772       String message = " Could not copy ESV histogram to backup - dynamics terminated.";
773       logger.log(Level.WARNING, message);
774       throw new RuntimeException(e);
775     }
776 
777     try {
778       Files.move(dyn.toPath(), dyn.toPath().resolveSibling(dynBackup.getName()),
779               StandardCopyOption.REPLACE_EXISTING);
780     } catch (IOException e) {
781       String message = " Could not copy dyn restart to backup - dynamics terminated.";
782       logger.log(Level.WARNING, message);
783       throw new RuntimeException(e);
784     }
785   }
786 
787   public int[] setTestingParametersAndExchangeOnce() {
788     // Copy parameters from validRestart directory
789     int[] temp = new int[]{2, 3, 0, 1, 6, 7, 4, 5};
790     System.arraycopy(temp, 0, pH2Rank, 0, temp.length);
791     System.arraycopy(temp, 0, rank2Ph, 0, temp.length);
792     double[][] temp1 = new double[][]{
793             {9.4, 43.26009292113161, 43.259888231035, 43.2596835409384, 43.259478850841795, 43.25927416074519, 43.25906947064859, 43.25886478055198, 43.25866009045538},
794             {9.9, 43.28675891594523, 43.286736736302245, 43.28671455665926, 43.28669237701629, 43.286670197373304, 43.28664801773032, 43.286625838087346, 43.28660365844436},
795             {8.4, 43.256007542834105, 43.25577488677315, 43.2555422307122, 43.25530957465124, 43.25507691859029, 43.25484426252933, 43.25461160646838, 43.25437895040743},
796             {8.9, 43.28985267712731, 43.28985166897153, 43.28985066081576, 43.28984965265999, 43.28984864450421, 43.28984763634844, 43.28984662819267, 43.28984562003689},
797             {11.4, 0.7237124623360065, 0.041741256298356136, -0.6402299497392943, -1.3222011557769446, -2.004172361814595, -2.686143567852245, -3.3681147738898956, -4.050085979927546},
798             {11.9, 0.7229705409794663, 0.0410229865156607, -0.6409245679481451, -1.3228721224119508, -2.0048196768757567, -2.6867672313395623, -3.368714785803368, -4.050662340267174},
799             {10.4, 0.7191692742214747, 0.037342984956673264, -0.6444833043081281, -1.3263095935729294, -2.0081358828377307, -2.689962172102532, -3.3717884613673337, -4.053614750632135},
800             {10.9, 0.6741663046798972, -0.006213315424885665, -0.6865929355296684, -1.3669725556344512, -2.047352175739234, -2.727731795844017, -3.4081114159487997, -4.088491036053583}
801     };
802     for(int i = 0; i < temp1.length; i++){
803       System.arraycopy(temp1[i], 0, parameters[i], 0, temp1[i].length);
804     }
805 
806     exchange();
807     int[] temp2 = new int[pH2Rank.length + rank2Ph.length];
808     System.arraycopy(pH2Rank, 0, temp2, 0, pH2Rank.length);
809     System.arraycopy(rank2Ph, 0, temp2, pH2Rank.length, rank2Ph.length);
810 
811     return temp2;
812   }
813 }