ryujin 2.1.1 revision 4c829a125920046d39a74afd45eabb6cc28a8170
checkpointing.h
Go to the documentation of this file.
1//
2// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
3// Copyright (C) 2020 - 2022 by the ryujin authors
4//
5
6#pragma once
7
9#include "offline_data.h"
10
11#include <deal.II/base/utilities.h>
12#include <deal.II/distributed/solution_transfer.h>
13
14#include <boost/archive/binary_iarchive.hpp>
15#include <boost/archive/binary_oarchive.hpp>
16#include <boost/core/demangle.hpp>
17
18#include <filesystem>
19#include <fstream>
20#include <string>
21
22namespace ryujin
23{
24 namespace Checkpointing
25 {
26 template <int dim>
28 std::is_same<typename Discretization<dim>::Triangulation,
29 dealii::parallel::distributed::Triangulation<dim>>::value;
30
38 template <int dim>
39 void load_mesh(Discretization<dim> &discretization,
40 const std::string &base_name)
41 {
42 if constexpr (have_distributed_triangulation<dim>) {
43 discretization.refinement() = 0; /* do not refine */
44 discretization.prepare();
45 discretization.triangulation().load(base_name + "-checkpoint.mesh");
46 } else {
47 AssertThrow(false, dealii::ExcNotImplemented());
48 __builtin_trap();
49 }
50 }
51
52
60 template <int dim, typename Number, int n_comp, int simd_length>
62 const OfflineData<dim, Number> &offline_data,
63 const std::string &base_name,
65 Number &t,
66 unsigned int &output_cycle,
67 const MPI_Comm &mpi_communicator)
68 {
69 if constexpr (have_distributed_triangulation<dim>) {
70 const auto &dof_handler = offline_data.dof_handler();
71
72 /* Create temporary scalar component vectors: */
73
74 const auto &scalar_partitioner = offline_data.scalar_partitioner();
75
77 std::array<ScalarVector, n_comp> state_vector;
78 for (auto &it : state_vector) {
79 it.reinit(scalar_partitioner);
80 }
81
82 /* Create SolutionTransfer object, attach state vector and deserialize:
83 */
84
85 dealii::parallel::distributed::SolutionTransfer<dim, ScalarVector>
86 solution_transfer(dof_handler);
87
88 std::vector<ScalarVector *> ptr_state;
89 std::transform(state_vector.begin(),
90 state_vector.end(),
91 std::back_inserter(ptr_state),
92 [](auto &it) { return &it; });
93
94 solution_transfer.deserialize(ptr_state);
95
96 unsigned int d = 0;
97 for (auto &it : state_vector) {
98 U.insert_component(it, d++);
99 }
100 U.update_ghost_values();
101
102 /* Read in and broadcast metadata: */
103
104 std::string name = base_name + "-checkpoint";
105
106 if (dealii::Utilities::MPI::this_mpi_process(mpi_communicator) == 0) {
107 std::string meta = name + ".metadata";
108
109 std::ifstream file(meta, std::ios::binary);
110 boost::archive::binary_iarchive ia(file);
111 ia >> t >> output_cycle;
112 }
113
114 int ierr;
115 if constexpr (std::is_same_v<Number, double>)
116 ierr = MPI_Bcast(&t, 1, MPI_DOUBLE, 0, mpi_communicator);
117 else
118 ierr = MPI_Bcast(&t, 1, MPI_FLOAT, 0, mpi_communicator);
119 AssertThrowMPI(ierr);
120
121 ierr = MPI_Bcast(&output_cycle, 1, MPI_UNSIGNED, 0, mpi_communicator);
122 AssertThrowMPI(ierr);
123
124 ierr = MPI_Barrier(mpi_communicator);
125 AssertThrowMPI(ierr);
126
127 } else {
128 AssertThrow(false, dealii::ExcNotImplemented());
129 __builtin_trap();
130 }
131 }
132
133
144 template <int dim, typename Number, int n_comp, int simd_length>
146 const OfflineData<dim, Number> &offline_data,
147 const std::string &base_name,
149 const Number t,
150 const unsigned int output_cycle,
151 const MPI_Comm &mpi_communicator)
152 {
153 if constexpr (have_distributed_triangulation<dim>) {
154 const auto &triangulation =
155 offline_data.discretization().triangulation();
156 const auto &dof_handler = offline_data.dof_handler();
157
158 /* Copy state into scalar component vectors: */
159
160 const auto &scalar_partitioner = offline_data.scalar_partitioner();
161
163 std::array<ScalarVector, n_comp> state_vector;
164 unsigned int d = 0;
165 for (auto &it : state_vector) {
166 it.reinit(scalar_partitioner);
167 U.extract_component(it, d++);
168 }
169
170 /* Create SolutionTransfer object, attach state vector and write out: */
171
172 dealii::parallel::distributed::SolutionTransfer<dim, ScalarVector>
173 solution_transfer(dof_handler);
174
175 std::vector<const ScalarVector *> ptr_state;
176 std::transform(state_vector.begin(),
177 state_vector.end(),
178 std::back_inserter(ptr_state),
179 [](auto &it) { return &it; });
180 solution_transfer.prepare_for_serialization(ptr_state);
181
182 std::string name = base_name + "-checkpoint";
183
184 if (dealii::Utilities::MPI::this_mpi_process(mpi_communicator) == 0) {
185 for (const std::string suffix :
186 {".mesh", ".mesh_fixed.data", ".mesh.info", ".metadata"})
187 if (std::filesystem::exists(name + suffix))
188 std::filesystem::rename(name + suffix, name + suffix + "~");
189 }
190
191 triangulation.save(name + ".mesh");
192
193 /* Metadata: */
194
195 if (dealii::Utilities::MPI::this_mpi_process(mpi_communicator) == 0) {
196 std::string meta = name + ".metadata";
197 std::ofstream file(meta, std::ios::binary | std::ios::trunc);
198 boost::archive::binary_oarchive oa(file);
199 oa << t << output_cycle;
200 }
201
202 const int ierr = MPI_Barrier(mpi_communicator);
203 AssertThrowMPI(ierr);
204
205 } else {
206 AssertThrow(false, dealii::ExcNotImplemented());
207 __builtin_trap();
208 }
209 }
210 } // namespace Checkpointing
211} // namespace ryujin
auto & triangulation() const
auto & dof_handler() const
Definition: offline_data.h:99
const auto & scalar_partitioner() const
Definition: offline_data.h:111
auto & discretization() const
Definition: offline_data.h:254
void extract_component(ScalarVector &scalar_vector, unsigned int component) const
void insert_component(const ScalarVector &scalar_vector, unsigned int component)
void load_state_vector(const OfflineData< dim, Number > &offline_data, const std::string &base_name, Vectors::MultiComponentVector< Number, n_comp, simd_length > &U, Number &t, unsigned int &output_cycle, const MPI_Comm &mpi_communicator)
Definition: checkpointing.h:61
void load_mesh(Discretization< dim > &discretization, const std::string &base_name)
Definition: checkpointing.h:39
void write_checkpoint(const OfflineData< dim, Number > &offline_data, const std::string &base_name, const Vectors::MultiComponentVector< Number, n_comp, simd_length > &U, const Number t, const unsigned int output_cycle, const MPI_Comm &mpi_communicator)
constexpr bool have_distributed_triangulation
Definition: checkpointing.h:27
dealii::LinearAlgebra::distributed::Vector< Number > ScalarVector
Definition: state_vector.h:25