1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 package ffx.algorithms.dynamics;
39
40 import static ffx.utilities.Constants.KCAL_TO_GRAM_ANG2_PER_PS2;
41 import static ffx.utilities.Constants.kB;
42 import static org.apache.commons.math3.util.FastMath.exp;
43
44 import edu.rit.mp.DoubleBuf;
45 import edu.rit.mp.IntegerBuf;
46 import edu.rit.mp.buf.IntegerMatrixBuf_1;
47 import edu.rit.pj.Comm;
48 import ffx.algorithms.Terminatable;
49 import ffx.numerics.Potential;
50 import ffx.potential.extended.ExtendedSystem;
51 import java.io.BufferedReader;
52 import java.io.File;
53 import java.io.FileReader;
54 import java.io.IOException;
55 import java.nio.file.Files;
56 import java.nio.file.StandardCopyOption;
57 import java.util.ArrayList;
58 import java.util.Arrays;
59 import java.util.List;
60 import java.util.Random;
61 import java.util.logging.Level;
62 import java.util.logging.Logger;
63
64 import ffx.potential.parameters.TitrationUtils;
65 import jline.internal.Nullable;
66 import org.apache.commons.io.FilenameUtils;
67
68
69
70
71
72
73
74
75 public class PhReplicaExchange implements Terminatable {
76
77 private static final Logger logger = Logger.getLogger(PhReplicaExchange.class.getName());
78 private final MolecularDynamics replica;
79 private final MolecularDynamicsOpenMM openMM;
80 private final Potential potential;
81 private final ExtendedSystem extendedSystem;
82 private final Random random;
83
84 private final Comm world;
85 private File dyn, esv, esvBackup, dynBackup;
86 private ArrayList<Double> readPhScale;
87
88
89
90
91 private final DoubleBuf[] parametersBuf;
92 private final DoubleBuf myParametersBuf;
93
94
95
96
97 private final double[][] parameters;
98 private final int[][][] parametersHis;
99 private final IntegerBuf[] parametersHisBuf;
100 private final int[] pH2Rank, rank2Ph, pHAcceptedCount, pHTrialCount;
101
102 private final int rank, nReplicas;
103 private boolean done = false, restart, terminate = false, backupNeeded = false;
104 private final double[] myParameters, pHScale;
105 private double[] x;
106 private final double pH, temp;
107 private int restartStep;
108
109
110
111
112
113
114
115
116
117
118 public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
119 double[] pHLadder, double temp, ExtendedSystem extendedSystem, double[] x) {
120 this(molecularDynamics, structureFile, pH, pHLadder, temp, extendedSystem, x, null, null);
121 }
122
123
124
125
126
127
128
129
130
131
132
133
134 public PhReplicaExchange(MolecularDynamics molecularDynamics, File structureFile, double pH,
135 double[] pHLadder, double temp, ExtendedSystem extendedSystem, @Nullable double[] x,
136 @Nullable MolecularDynamicsOpenMM molecularDynamicsOpenMM, @Nullable Potential potential) {
137
138 this.replica = molecularDynamics;
139 this.temp = temp;
140 this.extendedSystem = extendedSystem;
141 this.pH = pH;
142 this.pHScale = pHLadder;
143 this.x = x;
144 this.openMM = molecularDynamicsOpenMM;
145 this.potential = potential;
146
147
148 world = Comm.world();
149
150
151 int numProc = world.size();
152 rank = world.rank();
153
154 nReplicas = numProc;
155 pH2Rank = new int[nReplicas];
156 rank2Ph = new int[nReplicas];
157 pHAcceptedCount = new int[nReplicas];
158 pHTrialCount = new int[nReplicas];
159
160
161 for (int i = 0; i < nReplicas; i++) {
162 rank2Ph[i] = i;
163 pH2Rank[i] = i;
164 }
165 extendedSystem.setConstantPh(pHScale[rank]);
166
167 random = new Random();
168 random.setSeed(0);
169
170
171 parameters = new double[nReplicas][4];
172 parametersHis = new int[nReplicas][extendedSystem.getTitratingResidueList().size()][100];
173 parametersBuf = new DoubleBuf[nReplicas];
174 parametersHisBuf = new IntegerMatrixBuf_1[nReplicas];
175 for (int i = 0; i < nReplicas; i++) {
176 parametersBuf[i] = DoubleBuf.buffer(parameters[i]);
177 parametersHisBuf[i] = IntegerMatrixBuf_1.buffer(parametersHis[i]);
178 }
179
180
181 myParameters = parameters[rank];
182 myParametersBuf = parametersBuf[rank];
183
184
185 if (manageFilesAndRestart(structureFile)) {
186 logger.info(" Startup Successful! ");
187 extendedSystem.getESVHistogram(parametersHis[rank]);
188 } else {
189 logger.severe(" Unable to restart. Fix issues with restart files.");
190 }
191 }
192
193
194
195
196
197
198
199
200 private boolean manageFilesAndRestart(File structureFile) {
201
202 File parent = structureFile.getParentFile();
203 File rankDir = new File(parent + File.separator + rank);
204 restart = !rankDir.mkdir();
205
206
207 esv = new File(
208 rankDir.getPath() + File.separator + FilenameUtils.removeExtension(structureFile.getName())
209 + ".esv");
210 dyn = new File(
211 rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
212 + ".dyn");
213 esvBackup = new File(FilenameUtils.removeExtension(esv.getAbsolutePath()) + "_backup.esv");
214 dynBackup = new File(FilenameUtils.removeExtension(dyn.getAbsolutePath()) + "_backup.dyn");
215
216
217 extendedSystem.setRestartFile(esv);
218 replica.setFallbackDynFile(dyn);
219 replica.setArchiveFiles(new File[] {new File(
220 rankDir.getPath() + File.separator + FilenameUtils.removeExtension((structureFile.getName()))
221 + ".arc")});
222 if (openMM != null) {
223 openMM.setAutomaticWriteouts(false);
224 }
225
226
227 restartStep = 0;
228 if (restart) {
229 logger.info(" Attempting to restart. ");
230 readPhScale = new ArrayList<>();
231
232
233 backupNeeded = false;
234 if (checkForRestartFiles(parent, esv.getName()) && checkForRestartFiles(parent,
235 dyn.getName())) {
236 for (int i = 0; i < nReplicas; i++) {
237 File checkThisESV = new File(
238 parent.getAbsolutePath() + File.separator + i + File.separator + esv.getName());
239 backupNeeded = !readESV(checkThisESV, i);
240 if (backupNeeded) {
241 logger.warning(" Searching backup esv files.");
242 break;
243 }
244 }
245 if (!backupNeeded) {
246 logger.info(" Reading in from: " + esv);
247 extendedSystem.readESVInfoFrom(esv);
248 }
249 } else {
250 logger.info(" Directories do not contain all of the correct restart files.");
251 backupNeeded = true;
252 }
253
254
255 if (backupNeeded && checkForRestartFiles(parent, esvBackup.getName()) && checkForRestartFiles(
256 parent, dynBackup.getName())) {
257 for (int i = 0; i < nReplicas; i++) {
258 File checkThisESV = new File(
259 parent.getAbsolutePath() + File.separator + i + File.separator + esvBackup.getName());
260 if (!readESV(checkThisESV, i)) {
261 logger.info(" Restart files & Backups are messed up.");
262 return false;
263 }
264 }
265 logger.info(" Reading in from: " + esvBackup);
266 extendedSystem.readESVInfoFrom(esvBackup);
267 } else if (backupNeeded) {
268 logger.info(" Directories do not contain all of the correct backup restart files.");
269 logger.warning(" No backup files found, treating as a fresh start.");
270 restart = false;
271 return true;
272 }
273 } else {
274 logger.info(" Not a restart.");
275 return true;
276 }
277
278 logger.info(" Rank " + rank + " staring at pH " + pHScale[rank2Ph[rank]]);
279 extendedSystem.setConstantPh(pHScale[rank2Ph[rank]]);
280 logger.info(" Rank " + rank + " starting hist:");
281 extendedSystem.writeLambdaHistogram(true);
282
283 return true;
284 }
285
286
287
288
289
290
291
292
293
294
295 private boolean readESV(File file, int i) {
296 try (BufferedReader br = new BufferedReader(new FileReader(file))) {
297 String data = br.readLine();
298 while (data != null) {
299 List<String> tokens = Arrays.asList(data.split(" +"));
300 if (tokens.contains("pH:")) {
301 double pHOfRankI = Double.parseDouble(tokens.get(tokens.indexOf("pH:") + 1));
302
303
304 for (int j = 0; j < pHScale.length; j++) {
305 if (Math.abs(pHScale[j] / pHOfRankI - 1) < 0.00001) {
306 rank2Ph[i] = j;
307 pH2Rank[j] = i;
308 }
309 }
310
311
312 if (readPhScale.stream().anyMatch(d -> (Math.abs(d / pHOfRankI - 1) < 0.00001))) {
313 logger.warning(" Duplicate pH value found in file: " + file.getAbsolutePath());
314 readPhScale.clear();
315 restartStep = 0;
316 return false;
317 } else {
318 readPhScale.add(pHOfRankI);
319 }
320
321
322 br.readLine();
323 br.readLine();
324 int sum = 0;
325 for (int j = 0; j < 10; j++) {
326 data = br.readLine().trim();
327 tokens = List.of(data.split(" +"));
328 for (int k = 0; k < 10; k++) {
329 sum += Integer.parseInt(tokens.get(k + 1));
330 }
331 }
332
333 if (restartStep == 0) {
334 restartStep = sum;
335 } else if (restartStep != sum) {
336 logger.warning(" Restart received uneven sums from esv file: " + file.getAbsolutePath());
337 logger.info(" Restart Step: " + restartStep + " Found: " + sum);
338 restartStep = 0;
339 readPhScale.clear();
340 return false;
341 }
342 break;
343 }
344 data = br.readLine();
345 }
346 return true;
347 } catch (IOException | IndexOutOfBoundsException e) {
348 logger.warning("Failed to read file: " + file.getAbsolutePath());
349 return false;
350 }
351 }
352
353
354
355
356
357
358 private boolean checkForRestartFiles(File parent, String fileNameWithExtension) {
359 File searchFile;
360 for (int i = 0; i < nReplicas; i++) {
361 searchFile = new File(
362 parent.getAbsolutePath() + File.separator + i + File.separator + fileNameWithExtension);
363 if (!searchFile.exists()) {
364 return false;
365 }
366 }
367 return true;
368 }
369
370
371
372
373
374
375
376
377
378
379 public void sample(int cycles, long nSteps, double timeStep, double printInterval,
380 double trajInterval, int initTitrDynamics) {
381 sample(cycles, nSteps, 0, timeStep, printInterval, trajInterval, initTitrDynamics);
382 }
383
384 public void sample(int cycles, long titrSteps, long confSteps, double timeStep,
385 double printInterval, double trajInterval, int initDynamics) {
386 done = false;
387 terminate = false;
388 replica.setRestartFrequency(cycles * (titrSteps + confSteps) * replica.dt + 100);
389 extendedSystem.reGuessLambdas();
390 replica.setCoordinates(x);
391
392 int startCycle = 0;
393 if (initDynamics > 0 && !restart) {
394 extendedSystem.reGuessLambdas();
395 extendedSystem.setFixedTitrationState(true);
396 extendedSystem.setFixedTautomerState(true);
397
398 logger.info(" ");
399 logger.info(" ------------------Start of Equilibration Dynamics------------------\n");
400 logger.info(" ");
401
402 if (openMM == null) {
403 replica.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
404 } else {
405 x = replica.getCoordinates();
406 potential.energy(x);
407 openMM.setCoordinates(x);
408
409 openMM.dynamic(initDynamics, timeStep, printInterval, trajInterval, temp, true, dyn);
410
411 x = openMM.getCoordinates();
412 replica.setCoordinates(x);
413 }
414
415 extendedSystem.setFixedTitrationState(false);
416 extendedSystem.setFixedTautomerState(false);
417 logger.info(extendedSystem.getLambdaList());
418 extendedSystem.writeLambdaHistogram(true);
419 extendedSystem.copyESVHistogramTo(parametersHis[rank]);
420
421 logger.info(" ");
422 logger.info(" ------------------End of Equilibration Dynamics------------------\n");
423 logger.info(" ");
424 } else if (restart) {
425 logger.info(" Omitting initialization steps because this is a restart.");
426 startCycle = (int) (restartStep / titrSteps) + 1;
427 logger.info(" Restarting pH-REX at cycle " + startCycle + " of " + cycles);
428 }
429
430 for (int i = startCycle; i < cycles; i++) {
431
432 if (terminate) {
433 done = true;
434 break;
435 }
436
437 if (openMM != null) {
438 if (confSteps < 3) {
439 logger.severe(" Increase number of steps per cycle.");
440 }
441 dynamicsOpenMM(titrSteps, confSteps, timeStep, printInterval, trajInterval);
442 } else {
443 dynamics(titrSteps, timeStep, printInterval, trajInterval);
444 }
445
446 if (i == 0 || backupNeeded) {
447 replica.writeRestart();
448 }
449 copyToBackups();
450 replica.writeRestart();
451
452 if (i % 100 == 0) {
453 extendedSystem.writeLambdaHistogram(true);
454 }
455
456 try {
457 logger.info("\n Predicting pKa...");
458 logger.info(pkaPred(parametersHis));
459 } catch (Exception e) {
460 logger.info(e.getMessage());
461 logger.warning("Failed to predict pKa. Simulation may be too early on.");
462 }
463
464 logger.info(" ");
465 logger.info(String.format(" ------------------Exchange Cycle %d------------------\n", i + 1));
466
467 if (i != cycles - 1) {
468 exchange();
469 }
470 logger.info(" ");
471 logger.info(" Setting rank " + rank + " esv to pH " + pHScale[rank2Ph[rank]]);
472 logger.info(" ");
473 }
474 }
475
476 private String pkaPred(int[][][] parametersHis) {
477
478
479 double[][][] collapsedSum = new double[parametersHis.length][parametersHis[0].length][10];
480 double[][] collapsedRatio = new double[parametersHis.length][parametersHis[0].length];
481 for(int i = 0; i < parametersHis.length; i++)
482 {
483 for(int j = 0; j < parametersHis[0].length; j++)
484 {
485 for(int k = 0; k < 10; k++)
486 {
487 double sum = 0;
488 for(int l = 0; l < 10; l++)
489 {
490 sum += parametersHis[i][j][k*10 + l];
491 }
492 collapsedSum[i][j][k] = sum;
493 }
494
495 double ratio = collapsedSum[i][j][0] / (collapsedSum[i][j][0] + collapsedSum[i][j][9]);
496 collapsedRatio[i][j] = ratio;
497 }
498 }
499
500
501 StringBuilder output = new StringBuilder();
502 double[] n = new double[parametersHis[0].length];
503 double[] pka = new double[parametersHis[0].length];
504 double[][] residueRatios = new double[parametersHis[0].length][parametersHis.length];
505 for(int i = 0; i < parametersHis[0].length; i++)
506 {
507
508 for(int j = 0; j < parametersHis.length; j++)
509 {
510 residueRatios[i][j] = collapsedRatio[j][i];
511 }
512 Arrays.sort(residueRatios[i]);
513
514
515 double[] temp = TitrationUtils.predictHillCoeffandPka(pHScale, residueRatios[i]);
516 n[i] = temp[0];
517 pka[i] = temp[1];
518
519
520 String residueName = extendedSystem.getTitratingResidueList().get(i).toString();
521 output.append(" Residue: ").append(residueName).append("\n");
522 output.append(" Fractions (Dep / (Dep + Pro)): ").append(Arrays.toString(residueRatios[i])).append("\n");
523 output.append(" pH window: ").append(Arrays.toString(pHScale)).append("\n");
524 output.append(" n: ").append(n[i]).append("\n");
525 output.append(" pKa: ").append(pka[i]).append("\n");
526 output.append("\n");
527 }
528
529 return output.toString();
530 }
531
532
533
534
535 public static double[] setEvenSpacePhLadder(double pHGap, double pH, int nReplicas) {
536 double range = nReplicas * pHGap;
537 double pHMin = pH - range / 2;
538
539 if (nReplicas % 2 != 0) {
540 pHMin += pHGap / 2;
541 }
542
543 double[] pHScale = new double[nReplicas];
544 for (int i = 0; i < nReplicas; i++) {
545 pHScale[i] = pHMin + i * pHGap;
546 }
547
548 return pHScale;
549 }
550
551
552
553
554
555
556
557
558 @Override
559 public void terminate() {
560 terminate = true;
561 while (!done) {
562 synchronized (this) {
563 try {
564 wait(1);
565 } catch (InterruptedException e) {
566 logger.log(Level.WARNING, "Exception terminating replica exchange.\n", e);
567 }
568 }
569 }
570 }
571
572
573
574
575
576
577 private void compareTwo(int pH) {
578
579 int rankA;
580 int rankB;
581
582
583 double pHA;
584 double pHB;
585 double beta;
586 double acidostatA;
587 double acidostatB;
588 double acidostatAatB;
589 double acidostatBatA;
590
591 rankA = pH2Rank[pH];
592 rankB = pH2Rank[pH + 1];
593
594 pHA = parameters[rankA][0];
595 pHB = parameters[rankB][0];
596
597 beta = KCAL_TO_GRAM_ANG2_PER_PS2 / (temp * kB);
598
599 acidostatA = parameters[rankA][2];
600 acidostatB = parameters[rankB][2];
601
602 acidostatAatB = parameters[rankA][3];
603 acidostatBatA = parameters[rankB][1];
604
605 logger.info(" ");
606 logger.info(
607 " From rank " + rank + ": Comparing ranks " + rankA + " (pH = " + pHA + ") & " + rankB
608 + " (pH = " + pHB + ")");
609
610
611 double deltaE = beta * ((acidostatAatB + acidostatBatA) - (acidostatA + acidostatB));
612 logger.info(" DeltaE: " + deltaE);
613
614
615 pHTrialCount[pH]++;
616
617
618 if (deltaE < 0.0 || random.nextDouble() < exp(-deltaE)) {
619 int[][] tempHis = new int[extendedSystem.getTitratingResidueList().size()][100];
620 for (int i = 0; i < extendedSystem.getTitratingResidueList().size(); i++) {
621 System.arraycopy(parametersHis[rankA][i], 0, tempHis[i], 0, tempHis[i].length);
622 System.arraycopy(parametersHis[rankB][i], 0, parametersHis[rankA][i], 0,
623 parametersHis[rankA][i].length);
624 System.arraycopy(tempHis[i], 0, parametersHis[rankB][i], 0, parametersHis[rankB][i].length);
625 }
626
627 parameters[rankA][0] = pHB;
628 parameters[rankB][0] = pHA;
629
630
631 pH2Rank[pH] = rankB;
632 pH2Rank[pH + 1] = rankA;
633
634
635 rank2Ph[rankA] = pH + 1;
636 rank2Ph[rankB] = pH;
637
638 pHAcceptedCount[pH]++;
639 }
640 }
641
642
643 private void exchange() {
644
645 for (int pH = 0; pH < nReplicas - 1; pH++) {
646 compareTwo(pH);
647 }
648
649 logger.info(" ");
650
651 for (int i = 0; i < pHScale.length - 1; i++) {
652 double pHAcceptance = pHAcceptedCount[i] * 100.0 / (pHTrialCount[i]);
653 logger.info(
654 " Acceptance for pH " + pHScale[i] + " to be exchanged with pH " + pHScale[i + 1] + ": "
655 + pHAcceptance);
656 }
657 logger.info(" ");
658 }
659
660
661
662
663
664
665
666
667
668
669
670 private void dynamicsOpenMM(final long titrSteps, final long confSteps, final double timeStep,
671 final double printInterval, final double saveInterval) {
672
673 int i = rank2Ph[rank];
674 extendedSystem.setConstantPh(pHScale[i]);
675 extendedSystem.copyESVHistogramTo(parametersHis[rank]);
676
677
678 boolean initVelocities = true;
679
680 x = replica.getCoordinates();
681 potential.energy(x);
682 openMM.setCoordinates(x);
683
684 openMM.dynamic(confSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
685
686 x = openMM.getCoordinates();
687 replica.setCoordinates(x);
688
689 double forceWriteInterval = titrSteps * .001;
690 replica.dynamic(titrSteps, timeStep, printInterval, forceWriteInterval, temp, initVelocities, dyn);
691
692
693 myParameters[0] = pHScale[i];
694 myParameters[2] = extendedSystem.getBiasEnergy();
695
696 double lowPh;
697 double highPh;
698 try {
699 lowPh = pHScale[i - 1];
700 } catch (IndexOutOfBoundsException e) {
701 lowPh = 0;
702 }
703
704 try {
705 highPh = pHScale[i + 1];
706 } catch (IndexOutOfBoundsException e) {
707 highPh = 0;
708 }
709
710
711 extendedSystem.setConstantPh(lowPh);
712 myParameters[1] = extendedSystem.getBiasEnergy();
713
714 extendedSystem.setConstantPh(highPh);
715 myParameters[3] = extendedSystem.getBiasEnergy();
716
717 extendedSystem.setConstantPh(myParameters[0]);
718 extendedSystem.getESVHistogram(parametersHis[rank]);
719
720
721 try {
722 world.allGather(myParametersBuf, parametersBuf);
723 world.allGather(parametersHisBuf[rank], parametersHisBuf);
724 } catch (IOException ex) {
725 String message = " Replica Exchange allGather failed.";
726 logger.log(Level.SEVERE, message, ex);
727 }
728 }
729
730
731 private void dynamics(long nSteps, double timeStep, double printInterval, double saveInterval) {
732 int i = rank2Ph[rank];
733
734 extendedSystem.setConstantPh(pHScale[i]);
735
736 extendedSystem.copyESVHistogramTo(parametersHis[rank]);
737
738
739 boolean initVelocities = true;
740
741 replica.dynamic(nSteps, timeStep, printInterval, saveInterval, temp, initVelocities, dyn);
742
743
744 myParameters[0] = pHScale[i];
745 myParameters[2] = extendedSystem.getBiasEnergy();
746
747
748 double lowPh;
749 double highPh;
750 try {
751 lowPh = pHScale[i - 1];
752 } catch (IndexOutOfBoundsException e) {
753 lowPh = 0;
754 }
755
756 try {
757 highPh = pHScale[i + 1];
758 } catch (IndexOutOfBoundsException e) {
759 highPh = 0;
760 }
761
762 extendedSystem.setConstantPh(lowPh);
763 myParameters[1] = extendedSystem.getBiasEnergy();
764
765 extendedSystem.setConstantPh(highPh);
766 myParameters[3] = extendedSystem.getBiasEnergy();
767
768 extendedSystem.setConstantPh(myParameters[0]);
769
770 extendedSystem.getESVHistogram(parametersHis[rank]);
771
772
773 try {
774 world.allGather(myParametersBuf, parametersBuf);
775 world.allGather(parametersHisBuf[rank], parametersHisBuf);
776 } catch (IOException ex) {
777 String message = " Replica Exchange allGather failed.";
778 logger.log(Level.SEVERE, message, ex);
779 }
780 }
781
782 private void copyToBackups() {
783 try {
784 Files.move(esv.toPath(), esv.toPath().resolveSibling(esvBackup.getName()),
785 StandardCopyOption.REPLACE_EXISTING);
786 } catch (IOException e) {
787 String message = " Could not copy ESV histogram to backup - dynamics terminated.";
788 logger.log(Level.WARNING, message);
789 throw new RuntimeException(e);
790 }
791
792 try {
793 Files.move(dyn.toPath(), dyn.toPath().resolveSibling(dynBackup.getName()),
794 StandardCopyOption.REPLACE_EXISTING);
795 } catch (IOException e) {
796 String message = " Could not copy dyn restart to backup - dynamics terminated.";
797 logger.log(Level.WARNING, message);
798 throw new RuntimeException(e);
799 }
800 }
801 }