Chaste  Release::2018.1
ParallelColumnDataWriter.cpp
1 /*
2 
3 Copyright (c) 2005-2018, University of Oxford.
4 All rights reserved.
5 
6 University of Oxford means the Chancellor, Masters and Scholars of the
7 University of Oxford, having an administrative office at Wellington
8 Square, Oxford OX1 2JD, UK.
9 
10 This file is part of Chaste.
11 
12 Redistribution and use in source and binary forms, with or without
13 modification, are permitted provided that the following conditions are met:
14  * Redistributions of source code must retain the above copyright notice,
15  this list of conditions and the following disclaimer.
16  * Redistributions in binary form must reproduce the above copyright notice,
17  this list of conditions and the following disclaimer in the documentation
18  and/or other materials provided with the distribution.
19  * Neither the name of the University of Oxford nor the names of its
20  contributors may be used to endorse or promote products derived from this
21  software without specific prior written permission.
22 
23 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
29 GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
32 OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 
34 */
35 
36 #include "ParallelColumnDataWriter.hpp"
37 #include "Exception.hpp"
38 #include "DistributedVectorFactory.hpp"
39 
41  const std::string& rBaseName,
42  bool cleanDirectory)
43  : ColumnDataWriter(rDirectory, rBaseName, cleanDirectory),
44  mConcentrated(nullptr)
45 {
46  int num_procs;
47  MPI_Comm_size(PETSC_COMM_WORLD, &num_procs);
48  if (num_procs==1)
49  {
50  mIsParallel = false;
51  }
52  else
53  {
54  mIsParallel = true;
55  }
56 }
57 
58 void ParallelColumnDataWriter::PutVector(int variableID, Vec petscVector)
59 {
60  int size;
61  VecGetSize(petscVector,&size);
62 
63  if (size != mFixedDimensionSize)
64  {
65  EXCEPTION("Size of vector does not match FixedDimensionSize.");
66  }
67 
68  // Construct the appropriate "scatter" object to concentrate the vector on the master
69  if (mConcentrated==nullptr)
70  {
71  VecScatterCreateToZero(petscVector, &mToMaster, &mConcentrated);
72  }
73 
74 // int size2;
75 // VecGetSize(mConcentrated, &size2);
76 // std::cout << "Vector size=" << size << "," << size2 << std::endl << std::flush;
77 
78 //PETSc-3.x.x or PETSc-2.3.3
79 #if ((PETSC_VERSION_MAJOR == 3) || (PETSC_VERSION_MAJOR == 2 && PETSC_VERSION_MINOR == 3 && PETSC_VERSION_SUBMINOR == 3)) //2.3.3 or 3.x.x
80  VecScatterBegin(mToMaster, petscVector, mConcentrated, INSERT_VALUES, SCATTER_FORWARD);
81  VecScatterEnd(mToMaster, petscVector, mConcentrated, INSERT_VALUES, SCATTER_FORWARD);
82 #else
83  VecScatterBegin(petscVector, mConcentrated, INSERT_VALUES, SCATTER_FORWARD, mToMaster);
84  VecScatterEnd(petscVector, mConcentrated, INSERT_VALUES, SCATTER_FORWARD, mToMaster);
85 #endif
86 
87 // std::cout << "Done scatter" << std::endl << std::flush;
88 
90  {
91  double *concentrated_vector;
92  VecGetArray(mConcentrated, &concentrated_vector);
93  for (int i=0; i<size; i++)
94  {
95  ColumnDataWriter::PutVariable(variableID, concentrated_vector[i], i);
96  }
97  VecRestoreArray(mConcentrated, &concentrated_vector);
98  }
99 }
100 
102 {
103  // Put the stripe into its own 'unstriped' vector
104  DistributedVectorFactory* p_factory = rStripe.GetFactory();
105  Vec unstriped_petsc = p_factory->CreateVec();
106  DistributedVector unstriped = p_factory->CreateDistributedVector(unstriped_petsc);
107  for (DistributedVector::Iterator index = unstriped.Begin();
108  index!= unstriped.End();
109  ++index)
110  {
111  unstriped[index] = rStripe[index];
112  }
113 
114  // Put the unstriped vector
115  ParallelColumnDataWriter::PutVector(variableId, unstriped_petsc);
116  PetscTools::Destroy(unstriped_petsc);
117 }
118 
120 {
121  if (PetscTools::AmMaster())
122  {
124  }
125  else
126  {
127  mIsInDefineMode = false;
128  }
129 }
130 
139 void ParallelColumnDataWriter::PutVariable(int variableID, double variableValue, long dimensionPosition)
140 {
141  if (PetscTools::AmMaster())
142  {
143  // Master process is allowed to write
144  ColumnDataWriter::PutVariable(variableID, variableValue, dimensionPosition);
145  }
146 }
147 
149 {
150  if (mConcentrated != nullptr)
151  {
152  VecScatterDestroy(PETSC_DESTROY_PARAM(mToMaster));
154  }
155  Close();
156 }
157 
159 {
160  // Paranoia
161  PetscTools::Barrier("ParallelColumnDataWriter::AdvanceAlongUnlimitedDimension");
162 
163  if (PetscTools::AmMaster())
164  {
166  }
167 }
168 
170 {
171  // Paranoia
172  PetscTools::Barrier("ParallelColumnDataWriter::Close");
173 
174  if (PetscTools::AmMaster())
175  {
177  }
178 }
DistributedVectorFactory * GetFactory()
static void Barrier(const std::string callerId="")
Definition: PetscTools.cpp:134
virtual void PutVariable(int variableID, double variableValue, long dimensionPosition=-1)
#define PETSC_DESTROY_PARAM(x)
Definition: PetscTools.hpp:70
DistributedVector CreateDistributedVector(Vec vec, bool readOnly=false)
#define EXCEPTION(message)
Definition: Exception.hpp:143
static bool AmMaster()
Definition: PetscTools.cpp:120
void DoAdvanceAlongUnlimitedDimension()
virtual void Close()
virtual void EndDefineMode()
ParallelColumnDataWriter(const std::string &rDirectory, const std::string &rBaseName, bool cleanDirectory=true)
void PutVectorStripe(int variableId, DistributedVector::Stripe &rStripe)
void PutVariable(int variableID, double variableValue, long dimensionPosition=-1)
void PutVector(int variableID, Vec petscVector)
static void Destroy(Vec &rVec)
Definition: PetscTools.hpp:352