View Javadoc
1   // ******************************************************************************
2   //
3   // Title:       Force Field X.
4   // Description: Force Field X - Software for Molecular Biophysics.
5   // Copyright:   Copyright (c) Michael J. Schnieders 2001-2025.
6   //
7   // This file is part of Force Field X.
8   //
9   // Force Field X is free software; you can redistribute it and/or modify it
10  // under the terms of the GNU General Public License version 3 as published by
11  // the Free Software Foundation.
12  //
13  // Force Field X is distributed in the hope that it will be useful, but WITHOUT
14  // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
15  // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
16  // details.
17  //
18  // You should have received a copy of the GNU General Public License along with
19  // Force Field X; if not, write to the Free Software Foundation, Inc., 59 Temple
20  // Place, Suite 330, Boston, MA 02111-1307 USA
21  //
22  // Linking this library statically or dynamically with other modules is making a
23  // combined work based on this library. Thus, the terms and conditions of the
24  // GNU General Public License cover the whole combination.
25  //
26  // As a special exception, the copyright holders of this library give you
27  // permission to link this library with independent modules to produce an
28  // executable, regardless of the license terms of these independent modules, and
29  // to copy and distribute the resulting executable under terms of your choice,
30  // provided that you also meet, for each linked independent module, the terms
31  // and conditions of the license of that module. An independent module is a
32  // module which is not derived from or based on this library. If you modify this
33  // library, you may extend this exception to your version of the library, but
34  // you are not obligated to do so. If you do not wish to do so, delete this
35  // exception statement from your version.
36  //
37  // ******************************************************************************
38  package ffx.algorithms.dynamics;
39  
40  import edu.rit.mp.DoubleBuf;
41  import edu.rit.mp.IntegerBuf;
42  import edu.rit.mp.buf.IntegerMatrixBuf_1;
43  import edu.rit.pj.Comm;
44  import ffx.algorithms.Terminatable;
45  import ffx.numerics.Potential;
46  import ffx.potential.extended.ExtendedSystem;
47  import ffx.potential.parameters.TitrationUtils;
48  import org.apache.commons.io.FilenameUtils;
49  
50  import javax.annotation.Nullable;
51  import java.io.BufferedReader;
52  import java.io.File;
53  import java.io.FileReader;
54  import java.io.IOException;
55  import java.nio.file.Files;
56  import java.nio.file.StandardCopyOption;
57  import java.util.ArrayList;
58  import java.util.Arrays;
59  import java.util.List;
60  import java.util.Random;
61  import java.util.logging.Level;
62  import java.util.logging.Logger;
63  
64  import static ffx.utilities.Constants.KCAL_TO_GRAM_ANG2_PER_PS2;
65  import static ffx.utilities.Constants.kB;
66  import static org.apache.commons.math3.util.FastMath.exp;
67  
68  /**
69   * The PhReplicaExchange implements pH replica exchange. Adapted from "ReplicaExchange.java" by
70   * Timothy D. Fenn and Michael J. Schnieders
71   *
72   * @author Matthew Speranza and Andrew Thiel
73   * @since 1.0
74   */
75  public class PhReplicaExchange implements Terminatable {
76  
77    private static final Logger logger = Logger.getLogger(PhReplicaExchange.class.getName());
78    private final MolecularDynamics replica;
79    private final MolecularDynamicsOpenMM openMM;
80    private final Potential potential;
81    private final ExtendedSystem extendedSystem;
82    private final Random random;
83    /** Parallel Java world communicator. */
84    private final Comm world;
85    private File dyn, esv, esvBackup, dynBackup;
86    private ArrayList<Double> readPhScale;
87    /**
88     * Each parameter array is wrapped inside a Parallel Java DoubleBuf for the All-Gather
89     * communication calls.
90     */
91    private final DoubleBuf[] parametersBuf;
92    private final DoubleBuf myParametersBuf;
93    /**
94     * The parameters array stores communicated parameters for each process (i.e. each RepEx system).
95     * Currently, the array is of size [number of Processes][2].
96     */
97    private final double[][] parameters;
98    private final int[][][] parametersHis;
99    private final IntegerBuf[] parametersHisBuf;
100   private final int[] pH2Rank, rank2Ph, pHAcceptedCount, pHTrialCount;
101   /** Rank of this process. */
102   private final int rank, nReplicas;
103   private boolean done = false, restart, terminate = false, backupNeeded = false;
104   private final double[] myParameters, pHScale;
105   private double[] x;
106   private final double pH, temp;
107   private int restartStep;
108 
109   /**
110    * pHReplicaExchange constructor.
111    *
112    * @param molecularDynamics a {@link MolecularDynamics} object.
113    * @param pH pH = pKa will be changed from this initial value
114    * @param extendedSystem extended system attached to this process
115    * @param pHLadder range of pH's that replicas are created for
116    * @param temp temperature of replica
117    */
118   public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
119                            double[] pHLadder, double temp, ExtendedSystem extendedSystem, double[] x, int worldSize) throws Exception {
120     this(molecularDynamics, structureFile, pH, pHLadder, temp, extendedSystem, x, null, null, worldSize);
121   }
122 
123   /**
124    * OpenMM cycled pHReplicaExchange constructor.
125    *
126    * @param molecularDynamics a {@link MolecularDynamics} object.
127    * @param pH pH = pKa will be changed from this initial value
128    * @param extendedSystem extended system attached to this process
129    * @param pHLadder range of pH's that replicas are created for
130    * @param temp temperature of replica
131    * @param x array of coordinates
132    * @param molecularDynamicsOpenMM for running config steps on GPU
133    */
134   public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
135                            double[] pHLadder, double temp, ExtendedSystem extendedSystem, @Nullable double[] x,
136                            @Nullable MolecularDynamicsOpenMM molecularDynamicsOpenMM, @Nullable Potential potential,
137                            int worldSize) throws Exception {
138     this.replica = molecularDynamics;
139     this.temp = temp;
140     this.extendedSystem = extendedSystem;
141     this.pH = pH;
142     this.pHScale = pHLadder;
143     this.x = x;
144     this.openMM = molecularDynamicsOpenMM;
145     this.potential = potential;
146 
147     // Set up the Replica Exchange communication variables for Parallel Java communication between nodes.
148     world = Comm.world();
149 
150     // Number of processes is equal to the number of replicas.
151     int numProc = worldSize;
152     rank = world.rank();
153 
154     nReplicas = numProc;
155     pH2Rank = new int[nReplicas];
156     rank2Ph = new int[nReplicas];
157     pHAcceptedCount = new int[nReplicas];
158     pHTrialCount = new int[nReplicas];
159 
160     // Init map
161     for (int i = 0; i < nReplicas; i++) {
162       rank2Ph[i] = i;
163       pH2Rank[i] = i;
164     }
165     extendedSystem.setConstantPh(pHScale[rank]);
166 
167     random = new Random();
168     random.setSeed(0);
169 
170     // Create arrays to store the parameters of all processes.
171     parameters = new double[nReplicas][pHLadder.length + 1];
172     parametersHis = new int[nReplicas][extendedSystem.getTitratingResidueList().size()][100];
173     parametersBuf = new DoubleBuf[nReplicas];
174     parametersHisBuf = new IntegerMatrixBuf_1[nReplicas];
175     for (int i = 0; i < nReplicas; i++) {
176       parametersBuf[i] = DoubleBuf.buffer(parameters[i]);
177       parametersHisBuf[i] = IntegerMatrixBuf_1.buffer(parametersHis[i]);
178     }
179 
180     // A convenience reference to the parameters of this process are updated during communication calls.
181     myParameters = parameters[rank];
182     myParametersBuf = parametersBuf[rank];
183 
184     // Set the ESV that this rank will share has correct values after a restart
185     if (manageFilesAndRestart(structureFile)) {
186       logger.info(" Startup Successful! ");
187       extendedSystem.getESVHistogram(parametersHis[rank]);
188     } else {
189       throw new Exception("Failed to restart Replica Exchange.");
190     }
191   }
192 
193   /**
194    * This deals with anything having to do with restarts and backup files. Is identical between CPU
195    * and GPU.
196    * <p>
197    * Restart Rules:
198    * <p>
199    * 1. Checks for existence of primary restart files in all directories.
200    * <p>
201    * 2. Check for even sums of esv samples.
202    * <p>
203    * 3. Check for unique pH values in esv files.
204    * <p>
205    * 4. Fall back and repeat 1-3 with backup files if primary backup failed.
206    * <p>
207    * 5. Return true if all checks pass, false if any fail in both primary and backup.
208    * <p>
209    * Treats as fresh start if no restart files are found or if the primary are found, fail,
210    * and backups are not found.
211    *
212    * @param structureFile pdb file
213    * @return whether the restart was successful or not
214    */
215   public boolean manageFilesAndRestart(File structureFile) {
216     // My directory info
217     File parent = structureFile.getParentFile();
218     File rankDir = new File(parent + File.separator + rank);
219     restart = !rankDir.mkdir();
220 
221     // My files
222     esv = new File(
223             rankDir.getPath() + File.separator + FilenameUtils.removeExtension(structureFile.getName())
224                     + ".esv");
225     dyn = new File(
226             rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
227                     + ".dyn");
228     esvBackup = new File(FilenameUtils.removeExtension(esv.getAbsolutePath()) + "_backup.esv");
229     dynBackup = new File(FilenameUtils.removeExtension(dyn.getAbsolutePath()) + "_backup.dyn");
230 
231     // Set restart files - does not do any reading yet
232     extendedSystem.setRestartFile(esv);
233     replica.setFallbackDynFile(dyn);
234     replica.setArchiveFiles(new File[]{new File(
235             rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
236                     + ".arc")});
237     if (openMM != null) {
238       openMM.setAutomaticWriteouts(false);
239     }
240 
241     // Build restart
242     restartStep = 0;
243     if (restart) {
244       logger.info(" Attempting to restart. ");
245       readPhScale = new ArrayList<>();
246 
247       // Read normal restarts
248       backupNeeded = false;
249       if (checkForRestartFiles(parent, esv.getName()) && checkForRestartFiles(parent,
250               dyn.getName())) {
251         for (int i = 0; i < nReplicas; i++) {
252           File checkThisESV = new File(
253                   parent.getAbsolutePath() + File.separator + i + File.separator + esv.getName());
254           backupNeeded = !readESV(checkThisESV, i);
255           if (backupNeeded) {
256             logger.warning(" Searching backup esv files.");
257             break;
258           }
259         }
260         if (!backupNeeded) {
261           logger.info(" Reading in from: " + esv);
262           extendedSystem.readESVInfoFrom(esv);
263         }
264       } else {
265         logger.info(" Directories do not contain all of the correct restart files.");
266         backupNeeded = true;
267       }
268 
269       // Read backup restarts
270       if (backupNeeded && checkForRestartFiles(parent, esvBackup.getName()) && checkForRestartFiles(
271               parent, dynBackup.getName())) {
272         for (int i = 0; i < nReplicas; i++) {
273           File checkThisESV = new File(
274                   parent.getAbsolutePath() + File.separator + i + File.separator + esvBackup.getName());
275           if (!readESV(checkThisESV, i)) {
276             logger.info(" Restart files & Backups are messed up.");
277             return false;
278           }
279         }
280         logger.info(" Reading in from: " + esvBackup);
281         extendedSystem.readESVInfoFrom(esvBackup);
282       } else if (backupNeeded) {
283         logger.info(" Directories do not contain all of the correct backup restart files.");
284         logger.info(" No backup files found, treating as a fresh start.");
285         restart = false;
286         return true;
287       }
288     } else {
289       logger.info(" Not a restart.");
290       return true;
291     }
292 
293     logger.info(" Rank " + rank + " staring at pH " + pHScale[rank2Ph[rank]]);
294     extendedSystem.setConstantPh(pHScale[rank2Ph[rank]]);
295     logger.info(" Rank " + rank + " starting hist:");
296     extendedSystem.writeLambdaHistogram(true);
297 
298     return true;
299   }
300 
301   /**
302    * Reads an esv file to find the number of steps logged and if the pH matches the others in
303    * readPhScale. (N.B. - This only checks the first esv in the system for uneven sums, since it is
304    * assumed that all counts will be messed up if the first one is.)
305    *
306    * @param file to read
307    * @return if the esv file of this name at this rank works with the others, or if i=0, then it sets
308    *     the standards
309    */
310   private boolean readESV(File file, int i) {
311     try (BufferedReader br = new BufferedReader(new FileReader(file))) {
312       String data = br.readLine();
313       while (data != null) {
314         List<String> tokens = Arrays.asList(data.split(" +"));
315         if (tokens.contains("pH:")) {
316           double pHOfRankI = Double.parseDouble(tokens.get(tokens.indexOf("pH:") + 1));
317 
318           // Set up map
319           for (int j = 0; j < pHScale.length; j++) {
320             if (Math.abs(pHScale[j] / pHOfRankI - 1) < 0.00001) {
321               rank2Ph[i] = j;
322               pH2Rank[j] = i;
323             }
324           }
325 
326           // Check pH list for duplicates
327           if (readPhScale.stream().anyMatch(d -> (Math.abs(d / pHOfRankI - 1) < 0.00001))) {
328             logger.warning(" Duplicate pH value found in file: " + file.getAbsolutePath());
329             readPhScale.clear();
330             restartStep = 0;
331             return false;
332           } else {
333             readPhScale.add(pHOfRankI);
334           }
335 
336           // Find restart steps
337           br.readLine();
338           br.readLine();
339           int sum = 0;
340           for (int j = 0; j < 10; j++) { // 10 titr windows
341             data = br.readLine().trim();
342             tokens = List.of(data.split(" +"));
343             for (int k = 0; k < 10; k++) { // 10 tautomer windows
344               sum += Integer.parseInt(tokens.get(k + 1));
345             }
346           }
347 
348           if (restartStep == 0) {
349             restartStep = sum;
350           } else if (restartStep != sum) {
351             logger.warning(" Restart received uneven sums from esv file: " + file.getAbsolutePath());
352             logger.info(" Restart Step: " + restartStep + "    Found: " + sum);
353             restartStep = 0;
354             readPhScale.clear();
355             return false;
356           }
357           break;
358         }
359         data = br.readLine();
360       }
361       return true;
362     } catch (IOException | IndexOutOfBoundsException e) {
363       logger.warning("Failed to read file: " + file.getAbsolutePath());
364       return false;
365     }
366   }
367 
368   /**
369    * Looks through directories to see if they all contain the right file
370    *
371    * @return whether they all contain the correct file
372    */
373   private boolean checkForRestartFiles(File parent, String fileNameWithExtension) {
374     File searchFile;
375     for (int i = 0; i < nReplicas; i++) {
376       searchFile = new File(
377               parent.getAbsolutePath() + File.separator + i + File.separator + fileNameWithExtension);
378       if (!searchFile.exists()) {
379         return false;
380       }
381     }
382     return true;
383   }
384 
385   /**
386    * Sample. Restart file write-outs are entirely handled by this method based on how many steps are per cycle.
387    * User command input is ignored.
388    *
389    * @param cycles The number of cycles to run.
390    * @param nSteps The number of steps per cycle.
391    * @param timeStep The time step.
392    * @param printInterval The print interval.
393    */
394   public void sample(int cycles, long nSteps, double timeStep, double printInterval,
395                      double trajInterval, int initTitrDynamics) {
396     sample(cycles, nSteps, 0, timeStep, printInterval, trajInterval, initTitrDynamics);
397   }
398 
399   public void sample(int cycles, long titrSteps, long confSteps, double timeStep,
400                      double printInterval, double trajInterval, int initDynamics) {
401     done = false;
402     terminate = false;
403     replica.setRestartFrequency(cycles * (titrSteps + confSteps) * replica.dt + 100); // Full control over restarts handled by this class
404     replica.setCoordinates(x);
405     int startCycle = 0;
406     if (initDynamics > 0 && !restart) {
407       extendedSystem.reGuessLambdas();
408       extendedSystem.setFixedTitrationState(true);
409       extendedSystem.setFixedTautomerState(true);
410 
411       logger.info(" ");
412       logger.info(" ------------------Start of Equilibration Dynamics------------------\n");
413       logger.info(" ");
414 
415       if (openMM == null) {
416         replica.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
417       } else {
418         x = replica.getCoordinates();
419         potential.energy(x);
420         openMM.setCoordinates(x);
421 
422         openMM.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
423 
424         x = openMM.getCoordinates();
425         replica.setCoordinates(x);
426       }
427 
428       extendedSystem.setFixedTitrationState(false);
429       extendedSystem.setFixedTautomerState(false);
430       logger.info(extendedSystem.getLambdaList());
431       extendedSystem.writeLambdaHistogram(true);
432       extendedSystem.copyESVHistogramTo(parametersHis[rank]); // Copy the ESV hist to be empty
433 
434       logger.info(" ");
435       logger.info(" ------------------End of Equilibration Dynamics------------------\n");
436       logger.info(" ");
437     } else if (initDynamics == 0 && !restart){
438       extendedSystem.reGuessLambdas();
439     } else if (restart) {
440       logger.info(" Omitting initialization steps because this is a restart.");
441       startCycle = (int) (restartStep / titrSteps) + 1;
442       logger.info(" Restarting pH-REX at cycle " + startCycle + " of " + cycles);
443     }
444 
445     for (int i = startCycle; i < cycles; i++) {
446       // Check for termination request.
447       if (terminate) {
448         done = true;
449         break;
450       }
451 
452       if (openMM != null) {
453         if (confSteps < 3) {
454           logger.severe(" Increase number of steps per cycle.");
455         }
456         dynamicsOpenMM(titrSteps, confSteps, timeStep, printInterval, trajInterval);
457       } else {
458         dynamics(titrSteps, timeStep, printInterval, trajInterval);
459       }
460       // Set backups in case job is killed at bad time
461       if (i == 0 || backupNeeded) {
462         replica.writeRestart();
463       }
464       copyToBackups();
465       replica.writeRestart();
466 
467       if (i % 100 == 0) {
468         extendedSystem.writeLambdaHistogram(true);
469       }
470 
471       try {
472         logger.info("\n Predicting pKa...");
473         logger.info(pkaPred(parametersHis));
474       } catch (Exception e) {
475         logger.info(e.getMessage());
476         logger.warning("Failed to predict pKa. Simulation may be too early on.");
477       }
478 
479       logger.info(" ");
480       logger.info(String.format(" ------------------Exchange Cycle %d------------------\n", i + 1));
481 
482       if (i != cycles - 1) {
483         exchange();
484       }
485       logger.info(" ");
486       logger.info(" Setting rank " + rank + " esv to pH " + pHScale[rank2Ph[rank]]);
487       logger.info(" ");
488     }
489   }
490 
491   private String pkaPred(int[][][] parametersHis) {
492     // parametersHis shape explanation - [rank (w/ some pH)][residue][histogram --> len = 100]
493     // Calculate collapsed fractions of 10x10 histograms
494     double[][][] collapsedSum = new double[parametersHis.length][parametersHis[0].length][10];
495     double[][] collapsedRatio = new double[parametersHis.length][parametersHis[0].length];
496     for(int i = 0; i < parametersHis.length; i++) // pH's
497     {
498       for(int j = 0; j < parametersHis[0].length; j++) // residues
499       {
500         for(int k = 0; k < 10; k++) // 10 titration states
501         {
502           double sum = 0;
503           for(int l = 0; l < 10; l++) // 10 tautomer states
504           {
505             sum += parametersHis[i][j][k*10 + l];
506           }
507           collapsedSum[i][j][k] = sum;
508         }
509         // Calculate ratio over 0 and 9
510         double ratio = collapsedSum[i][j][0] / (collapsedSum[i][j][0] + collapsedSum[i][j][9]);
511         collapsedRatio[i][j] = ratio;
512       }
513     }
514 
515     // Calculate hill coeff and pKa's of each residue and print
516     StringBuilder output = new StringBuilder();
517     double[] n = new double[parametersHis[0].length];
518     double[] pka = new double[parametersHis[0].length];
519     double[][] residueRatios = new double[parametersHis[0].length][parametersHis.length];
520     for(int i = 0; i < parametersHis[0].length; i++) // residues
521     {
522       // Create array of fractions for this residue
523       for(int j = 0; j < parametersHis.length; j++) // pH's
524       {
525         residueRatios[i][j] = collapsedRatio[j][i];
526       }
527       //Arrays.sort(residueRatios[i]);
528 
529       // L-BFGS minimization of the Henderson-Hasselbalch equation to find the best fit hill coeff and pKa
530       // Create Array of pH to match ordering residueRatios
531       double[] pHArray = new double[nReplicas];
532       for(int j=0; j < nReplicas; j++){
533         pHArray[j] = parameters[j][0];
534       }
535       double[] temp = TitrationUtils.predictHillCoeffandPka(pHArray, residueRatios[i]);
536       n[i] = temp[0];
537       pka[i] = temp[1];
538 
539       // Print results for this residue
540       String residueName = extendedSystem.getTitratingResidueList().get(i).toString();
541       output.append(" Residue: ").append(residueName).append("\n");
542       output.append(" Fractions (Dep / (Dep + Pro)): ").append(Arrays.toString(residueRatios[i])).append("\n");
543       output.append(" pH window: ").append(Arrays.toString(pHArray)).append("\n");
544       output.append(" n: ").append(n[i]).append("\n");
545       output.append(" pKa: ").append(pka[i]).append("\n");
546       output.append("\n");
547     }
548 
549     return output.toString();
550   }
551 
552   /**
553    * Sets an even pH ladder based on the pH gap.
554    */
555   public static double[] setEvenSpacePhLadder(double pHGap, double pH, int nReplicas) {
556     double range = nReplicas * pHGap;
557     double pHMin = pH - range / 2;
558 
559     if (nReplicas % 2 != 0) {
560       pHMin += pHGap / 2; // Center range if odd num windows
561     }
562 
563     double[] pHScale = new double[nReplicas];
564     for (int i = 0; i < nReplicas; i++) {
565       pHScale[i] = pHMin + i * pHGap;
566     }
567 
568     return pHScale;
569   }
570 
571   /**
572    * Get the pH scale array.
573    *
574    * @return the pH scale array
575    */
576   public double[] getPhScale() {
577     return pHScale;
578   }
579 
580   /**
581    * {@inheritDoc}
582    *
583    * <p>This should be implemented as a blocking interrupt; when the method returns the <code>
584    * Terminatable</code> algorithm has reached a clean termination point. For example, between
585    * minimize or molecular dynamics steps.
586    */
587   @Override
588   public void terminate() {
589     terminate = true;
590     while (!done) {
591       synchronized (this) {
592         try {
593           wait(1);
594         } catch (InterruptedException e) {
595           logger.log(Level.WARNING, "Exception terminating replica exchange.\n", e);
596         }
597       }
598     }
599   }
600 
601   /**
602    * Evaluate whether to exchange.
603    *
604    * @param pH The pH to have as the replica target.
605    */
606   private void compareTwo(int pH) {
607     // Ranks for pH A and B
608     int rankA;
609     int rankB;
610 
611     // Load pH, beta and energy for each rank.
612     double pHA;
613     double pHB;
614 
615     double beta;
616     double acidostatA;
617     double acidostatB;
618     double acidostatAatB;
619     double acidostatBatA;
620 
621     rankA = pH2Rank[pH];
622     rankB = pH2Rank[pH + 1];
623     int acidAIndex = rank2Ph[rankA] + 1;
624     int acidBIndex = rank2Ph[rankB] + 1;
625 
626     pHA = parameters[rankA][0];
627     pHB = parameters[rankB][0];
628 
629     beta = KCAL_TO_GRAM_ANG2_PER_PS2 / (temp * kB);
630 
631     acidostatA = parameters[rankA][acidAIndex];
632     acidostatB = parameters[rankB][acidBIndex];
633 
634     acidostatAatB = parameters[rankA][acidBIndex]; // acidostat of rankA evaluated at the pH of rankB
635     acidostatBatA = parameters[rankB][acidAIndex];
636 
637     logger.info(" ");
638     logger.info(
639             " From rank " + rank + ": Comparing ranks " + rankA + " (pH = " + pHA + ") & " + rankB
640                     + " (pH = " + pHB + ")");
641 
642     // Compute the change in energy over kT (E/kT) for the Metropolis criteria.
643     double deltaE = beta * ((acidostatAatB + acidostatBatA) - (acidostatA + acidostatB));
644     logger.info("pHA = " + pHA + " " + acidostatA + " " + acidostatAatB);
645     logger.info("pHB = " + pHB + " " + acidostatB + " " + acidostatBatA);
646     logger.info("exp(" + beta + " * ((" + acidostatAatB + " + " + acidostatBatA + ") - (" + acidostatA + " + " + acidostatB + ")))");
647     logger.info(" DeltaE: " + deltaE);
648 
649     //Count the number of trials for each temp
650     pHTrialCount[pH]++;
651 
652     // If the Metropolis criteria is satisfied, do the switch.
653     if (deltaE < 0.0 || random.nextDouble() < exp(-deltaE)) {
654       int[][] tempHis = new int[extendedSystem.getTitratingResidueList().size()][100];
655       for (int i = 0; i < extendedSystem.getTitratingResidueList().size(); i++) {
656         System.arraycopy(parametersHis[rankA][i], 0, tempHis[i], 0, tempHis[i].length);
657         System.arraycopy(parametersHis[rankB][i], 0, parametersHis[rankA][i], 0,
658                 parametersHis[rankA][i].length);
659         System.arraycopy(tempHis[i], 0, parametersHis[rankB][i], 0, parametersHis[rankB][i].length);
660       }
661       parameters[rankA][0] = pHB;
662       parameters[rankB][0] = pHA;
663 
664       //Log the new parameters
665       logger.info(" pH After swap " + parameters[rankA][0] + " rankA " + rankA + " acidostat: " + Arrays.toString(parameters[rankA]));
666       logger.info(" pH After swap " + parameters[rankB][0] + " rankB " + rankB + " acidostat: " + Arrays.toString(parameters[rankB]));
667 
668       // Map pH to process ranks.
669       pH2Rank[pH] = rankB;
670       pH2Rank[pH + 1] = rankA;
671 
672       // Map ranks to pH.
673       rank2Ph[rankA] = pH + 1;
674       rank2Ph[rankB] = pH;
675 
676       pHAcceptedCount[pH]++;
677     }
678   }
679 
680   /** All processes complete the exchanges identically given the same Random number seed. */
681   private void exchange() {
682     // Loop over top and bottom parts of pH scale
683     for (int pH = 0; pH < nReplicas - 1; pH++) {
684       compareTwo(pH);
685     }
686 
687     logger.info(" ");
688     // Print Exchange Info
689     for (int i = 0; i < pHScale.length - 1; i++) {
690       double pHAcceptance = pHAcceptedCount[i] * 100.0 / (pHTrialCount[i]);
691       logger.info(
692               " Acceptance for pH " + pHScale[i] + " to be exchanged with pH " + pHScale[i + 1] + ": "
693                       + pHAcceptance);
694     }
695     logger.info(" ");
696   }
697 
698   /**
699    * Blocking dynamic steps: when this method returns each replica has completed the requested number
700    * of steps. Both OpenMM and CPU implementations exist
701    *
702    * @param titrSteps the number of time steps on CPU.
703    * @param confSteps the number of time steps on GPU.
704    * @param timeStep the time step.
705    * @param printInterval the number of steps between logging updates.
706    * @param saveInterval the number of steps between saving snapshots.
707    */
708   private void dynamicsOpenMM(final long titrSteps, final long confSteps, final double timeStep,
709                               final double printInterval, final double saveInterval) {
710 
711     int i = rank2Ph[rank];
712     // Update this ranks' parameter array to be consistent with the dynamics.
713     myParameters[0] = pHScale[i];
714     extendedSystem.setConstantPh(myParameters[0]);
715     extendedSystem.copyESVHistogramTo(parametersHis[rank]);
716 
717     // Start this processes MolecularDynamics instance sampling.
718     boolean initVelocities = true;
719 
720     x = replica.getCoordinates();
721     potential.energy(x);
722     openMM.setCoordinates(x);
723     
724     openMM.dynamic(confSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
725     x = openMM.getCoordinates();
726     replica.setCoordinates(x);
727 
728     double forceWriteInterval = titrSteps * .001;
729     replica.dynamic(titrSteps, timeStep, printInterval, forceWriteInterval, temp, initVelocities, dyn);
730     reEvaulateAcidostats();
731 
732     extendedSystem.getESVHistogram(parametersHis[rank]);
733 
734     // Gather all parameters from the other processes.
735     try {
736       world.allGather(myParametersBuf, parametersBuf);
737       world.allGather(parametersHisBuf[rank], parametersHisBuf);
738     } catch (IOException ex) {
739       String message = " Replica Exchange allGather failed.";
740       logger.log(Level.SEVERE, message, ex);
741     }
742   }
743 
744 
745   private void dynamics(long nSteps, double timeStep, double printInterval, double saveInterval) {
746     int i = rank2Ph[rank];
747 
748     // Update this ranks' parameter array to be consistent with the dynamics.
749     myParameters[0] = pHScale[i];
750     extendedSystem.setConstantPh(myParameters[0]);
751 
752     extendedSystem.copyESVHistogramTo(parametersHis[rank]);
753 
754     // Start this processes MolecularDynamics instance sampling.
755     boolean initVelocities = true;
756 
757     replica.dynamic(nSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
758 
759     reEvaulateAcidostats();
760 
761     extendedSystem.getESVHistogram(parametersHis[rank]);
762 
763     // Gather all parameters from the other processes.
764     try {
765       world.allGather(myParametersBuf, parametersBuf);
766       world.allGather(parametersHisBuf[rank], parametersHisBuf);
767     } catch (IOException ex) {
768       String message = " Replica Exchange allGather failed.";
769       logger.log(Level.SEVERE, message, ex);
770     }
771   }
772 
773   private void reEvaulateAcidostats() {
774     for(int j = 0; j < pHScale.length; j++){
775       extendedSystem.setConstantPh(pHScale[j]); // A at A+gap(B)
776       myParameters[1+j] = extendedSystem.getBiasEnergy();
777     }
778 
779     extendedSystem.setConstantPh(myParameters[0]);
780   }
781 
782   private void copyToBackups() {
783     try {
784       Files.move(esv.toPath(), esv.toPath().resolveSibling(esvBackup.getName()),
785               StandardCopyOption.REPLACE_EXISTING);
786     } catch (IOException e) {
787       String message = " Could not copy ESV histogram to backup - dynamics terminated.";
788       logger.log(Level.WARNING, message);
789       throw new RuntimeException(e);
790     }
791 
792     try {
793       Files.move(dyn.toPath(), dyn.toPath().resolveSibling(dynBackup.getName()),
794               StandardCopyOption.REPLACE_EXISTING);
795     } catch (IOException e) {
796       String message = " Could not copy dyn restart to backup - dynamics terminated.";
797       logger.log(Level.WARNING, message);
798       throw new RuntimeException(e);
799     }
800   }
801 
802   public int[] setTestingParametersAndExchangeOnce() {
803     // Copy parameters from validRestart directory
804     int[] temp = new int[]{2, 3, 0, 1, 6, 7, 4, 5};
805     System.arraycopy(temp, 0, pH2Rank, 0, temp.length);
806     System.arraycopy(temp, 0, rank2Ph, 0, temp.length);
807     double[][] temp1 = new double[][]{
808             {9.4, 43.26009292113161, 43.259888231035, 43.2596835409384, 43.259478850841795, 43.25927416074519, 43.25906947064859, 43.25886478055198, 43.25866009045538},
809             {9.9, 43.28675891594523, 43.286736736302245, 43.28671455665926, 43.28669237701629, 43.286670197373304, 43.28664801773032, 43.286625838087346, 43.28660365844436},
810             {8.4, 43.256007542834105, 43.25577488677315, 43.2555422307122, 43.25530957465124, 43.25507691859029, 43.25484426252933, 43.25461160646838, 43.25437895040743},
811             {8.9, 43.28985267712731, 43.28985166897153, 43.28985066081576, 43.28984965265999, 43.28984864450421, 43.28984763634844, 43.28984662819267, 43.28984562003689},
812             {11.4, 0.7237124623360065, 0.041741256298356136, -0.6402299497392943, -1.3222011557769446, -2.004172361814595, -2.686143567852245, -3.3681147738898956, -4.050085979927546},
813             {11.9, 0.7229705409794663, 0.0410229865156607, -0.6409245679481451, -1.3228721224119508, -2.0048196768757567, -2.6867672313395623, -3.368714785803368, -4.050662340267174},
814             {10.4, 0.7191692742214747, 0.037342984956673264, -0.6444833043081281, -1.3263095935729294, -2.0081358828377307, -2.689962172102532, -3.3717884613673337, -4.053614750632135},
815             {10.9, 0.6741663046798972, -0.006213315424885665, -0.6865929355296684, -1.3669725556344512, -2.047352175739234, -2.727731795844017, -3.4081114159487997, -4.088491036053583}
816     };
817     for(int i = 0; i < temp1.length; i++){
818       System.arraycopy(temp1[i], 0, parameters[i], 0, temp1[i].length);
819     }
820 
821     exchange();
822     int[] temp2 = new int[pH2Rank.length + rank2Ph.length];
823     System.arraycopy(pH2Rank, 0, temp2, 0, pH2Rank.length);
824     System.arraycopy(rank2Ph, 0, temp2, pH2Rank.length, rank2Ph.length);
825 
826     return temp2;
827   }
828 }