simulator.h

Go to the documentation of this file.

/*
  Copyright (C) 2011 - 2022 by the authors of the ASPECT code.

  This file is part of ASPECT.

  ASPECT is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2, or (at your option)
  any later version.

  ASPECT is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with ASPECT; see the file LICENSE.  If not see
  <http://www.gnu.org/licenses/>.
*/


#ifndef _aspect_simulator_h
#define _aspect_simulator_h

#include <deal.II/base/timer.h>
#include <deal.II/base/parameter_handler.h>
#include <deal.II/base/conditional_ostream.h>
#include <deal.II/base/symmetric_tensor.h>

DEAL_II_DISABLE_EXTRA_DIAGNOSTICS

#include <deal.II/lac/affine_constraints.h>

#include <deal.II/distributed/tria.h>

#include <deal.II/dofs/dof_handler.h>
#include <deal.II/dofs/dof_tools.h>

#include <deal.II/fe/fe_system.h>
#include <deal.II/fe/mapping.h>
#include <deal.II/base/tensor_function.h>

DEAL_II_ENABLE_EXTRA_DIAGNOSTICS

#include <aspect/global.h>
#include <aspect/simulator_access.h>
#include <aspect/lateral_averaging.h>
#include <aspect/simulator_signals.h>
#include <aspect/material_model/interface.h>
#include <aspect/heating_model/interface.h>
#include <aspect/geometry_model/initial_topography_model/interface.h>
#include <aspect/geometry_model/interface.h>
#include <aspect/gravity_model/interface.h>
#include <aspect/boundary_temperature/interface.h>
#include <aspect/boundary_heat_flux/interface.h>
#include <aspect/boundary_composition/interface.h>
#include <aspect/initial_temperature/interface.h>
#include <aspect/initial_composition/interface.h>
#include <aspect/prescribed_stokes_solution/interface.h>
#include <aspect/boundary_velocity/interface.h>
#include <aspect/boundary_fluid_pressure/interface.h>
#include <aspect/boundary_traction/interface.h>
#include <aspect/mesh_refinement/interface.h>
#include <aspect/time_stepping/interface.h>
#include <aspect/postprocess/interface.h>
#include <aspect/adiabatic_conditions/interface.h>
#include <aspect/particle/world.h>

#include <boost/iostreams/tee.hpp>
#include <boost/iostreams/stream.hpp>

#include <memory>
#include <thread>


namespace aspect
{
  using namespace dealii;

  template <int dim>
  class MeltHandler;

  template <int dim>
  class NewtonHandler;

  template <int dim>
  class StokesMatrixFreeHandler;

  template <int dim, int velocity_degree>
  class StokesMatrixFreeHandlerImplementation;

  namespace MeshDeformation
  {
    template <int dim>
    class MeshDeformationHandler;
  }

  template <int dim>
  class VolumeOfFluidHandler;

  namespace internal
  {
    namespace Assembly
    {
      namespace Scratch
      {
        template <int dim>      struct StokesPreconditioner;
        template <int dim>      struct StokesSystem;
        template <int dim>      struct AdvectionSystem;
      }

      namespace CopyData
      {
        template <int dim>      struct StokesPreconditioner;
        template <int dim>      struct StokesSystem;
        template <int dim>      struct AdvectionSystem;
      }
    }
  }

  namespace Assemblers
  {
    template <int dim>      class Interface;
    template <int dim>      class Manager;
  }

  struct DefectCorrectionResiduals
  {
    double initial_residual;
    double velocity_residual;
    double pressure_residual;
    double residual;
    double residual_old;
    double switch_initial_residual;
    double newton_residual_for_derivative_scaling_factor;
    std::pair<double,double> stokes_residuals;
  };

  template <int dim>
  struct RotationProperties
  {
    RotationProperties()
      :
      scalar_moment_of_inertia(numbers::signaling_nan<double>()),
      scalar_angular_momentum(numbers::signaling_nan<double>()),
      scalar_rotation(numbers::signaling_nan<double>()),
      tensor_moment_of_inertia(numbers::signaling_nan<SymmetricTensor<2,dim>>()),
      tensor_angular_momentum(numbers::signaling_nan<Tensor<1,dim>>()),
      tensor_rotation(numbers::signaling_nan<Tensor<1,dim>>())
    {};

    double scalar_moment_of_inertia;
    double scalar_angular_momentum;
    double scalar_rotation;

    SymmetricTensor<2,dim> tensor_moment_of_inertia;
    Tensor<1,dim> tensor_angular_momentum;
    Tensor<1,dim> tensor_rotation;
  };

  template <int dim>
  class Simulator
  {
    public:
      Simulator (const MPI_Comm mpi_communicator,
                 ParameterHandler &prm);

      ~Simulator ();

      static
      void declare_parameters (ParameterHandler &prm);

      void run ();

      void
      write_plugin_graph (std::ostream &output_stream) const;

      using NonlinearSolver = typename Parameters<dim>::NonlinearSolver;

      using NullspaceRemoval = typename Parameters<dim>::NullspaceRemoval;


      struct AdvectionField
      {
        enum FieldType { temperature_field, compositional_field };

        const FieldType    field_type;

        const unsigned int compositional_variable;

        AdvectionField (const FieldType field_type,
                        const unsigned int compositional_variable = numbers::invalid_unsigned_int);

        static
        AdvectionField temperature ();

        static
        AdvectionField composition (const unsigned int compositional_variable);

        bool
        is_temperature () const;

        bool
        is_discontinuous (const Introspection<dim> &introspection) const;

        typename Parameters<dim>::AdvectionFieldMethod::Kind
        advection_method (const Introspection<dim> &introspection) const;

        unsigned int component_index(const Introspection<dim> &introspection) const;

        unsigned int block_index(const Introspection<dim> &introspection) const;

        unsigned int field_index() const;

        unsigned int base_element(const Introspection<dim> &introspection) const;

        FEValuesExtractors::Scalar scalar_extractor(const Introspection<dim> &introspection) const;

        unsigned int polynomial_degree(const Introspection<dim> &introspection) const;
      };

    private:

      bool simulator_is_past_initialization;

      struct IntermediaryConstructorAction
      {
        IntermediaryConstructorAction (const std::function<void ()> &action);
      };

      void setup_dofs ();

      void setup_introspection ();

      void set_initial_temperature_and_compositional_fields ();

      void compute_initial_pressure_field ();

      void compute_initial_velocity_boundary_constraints (AffineConstraints<double> &constraints);

      void compute_current_velocity_boundary_constraints (AffineConstraints<double> &constraints);

      void compute_current_constraints ();

      double compute_pressure_scaling_factor () const;

      void start_timestep ();

      void solve_timestep ();

      void solve_single_advection_single_stokes ();

      void solve_no_advection_iterated_stokes ();

      void solve_no_advection_single_stokes ();

      void solve_first_timestep_only_single_stokes ();

      void solve_iterated_advection_and_stokes ();

      void solve_single_advection_iterated_stokes ();

      void solve_no_advection_iterated_defect_correction_stokes ();

      void solve_single_advection_iterated_defect_correction_stokes ();

      void solve_iterated_advection_and_defect_correction_stokes ();

      void solve_iterated_advection_and_newton_stokes ();

      void solve_single_advection_iterated_newton_stokes ();

      void solve_single_advection_no_stokes ();

      void solve_no_advection_no_stokes ();

      void build_stokes_preconditioner ();

      void build_advection_preconditioner (const AdvectionField &advection_field,
                                           aspect::LinearAlgebra::PreconditionILU &preconditioner,
                                           const double diagonal_strengthening);

      void assemble_stokes_system ();

      double assemble_and_solve_temperature (const bool compute_initial_residual = false,
                                             double *initial_residual = nullptr);

      std::vector<double> assemble_and_solve_composition (const bool compute_initial_residual = false,
                                                          std::vector<double> *initial_residual = nullptr);

      double assemble_and_solve_stokes (const bool compute_initial_residual = false,
                                        double *initial_nonlinear_residual = nullptr);

      void assemble_and_solve_defect_correction_Stokes(DefectCorrectionResiduals &dcr,
                                                       const bool use_picard);

      void assemble_advection_system (const AdvectionField &advection_field);

      double solve_advection (const AdvectionField &advection_field);

      void interpolate_particle_properties (const AdvectionField &advection_field);

      std::pair<double,double>
      solve_stokes ();

      std::pair<double,double>
      solve_stokes_block_gmg ();

      void postprocess ();

      void refine_mesh (const unsigned int max_grid_level);

      void create_snapshot();

      void resume_from_snapshot();

      template <class Archive>
      void serialize (Archive &ar, const unsigned int version);
      Table<2,DoFTools::Coupling>
      setup_system_matrix_coupling () const;

      void setup_system_matrix (const std::vector<IndexSet> &system_partitioning);

      void setup_system_preconditioner (const std::vector<IndexSet> &system_partitioning);

      std::unique_ptr<Assemblers::Manager<dim>> assemblers;

      void set_assemblers ();

      void set_advection_assemblers ();

      void set_stokes_assemblers ();

      void assemble_stokes_preconditioner ();

      void
      local_assemble_stokes_preconditioner (const typename DoFHandler<dim>::active_cell_iterator &cell,
                                            internal::Assembly::Scratch::StokesPreconditioner<dim> &scratch,
                                            internal::Assembly::CopyData::StokesPreconditioner<dim> &data);

      void
      copy_local_to_global_stokes_preconditioner (const internal::Assembly::CopyData::StokesPreconditioner<dim> &data);

      void
      local_assemble_stokes_system (const typename DoFHandler<dim>::active_cell_iterator &cell,
                                    internal::Assembly::Scratch::StokesSystem<dim>  &scratch,
                                    internal::Assembly::CopyData::StokesSystem<dim> &data);

      void
      copy_local_to_global_stokes_system (const internal::Assembly::CopyData::StokesSystem<dim> &data);

      void
      local_assemble_advection_face_terms(const AdvectionField &advection_field,
                                          const typename DoFHandler<dim>::active_cell_iterator &cell,
                                          internal::Assembly::Scratch::AdvectionSystem<dim> &scratch,
                                          internal::Assembly::CopyData::AdvectionSystem<dim> &data);
      void
      local_assemble_advection_system (const AdvectionField &advection_field,
                                       const Vector<double>           &viscosity_per_cell,
                                       const typename DoFHandler<dim>::active_cell_iterator &cell,
                                       internal::Assembly::Scratch::AdvectionSystem<dim>  &scratch,
                                       internal::Assembly::CopyData::AdvectionSystem<dim> &data);

      void
      copy_local_to_global_advection_system (const AdvectionField &advection_field,
                                             const internal::Assembly::CopyData::AdvectionSystem<dim> &data);

      void make_pressure_rhs_compatible(LinearAlgebra::BlockVector &vector);

      template <typename T>
      void get_artificial_viscosity (Vector<T> &viscosity_per_cell,
                                     const AdvectionField &advection_field,
                                     const bool skip_interior_cells = false) const;

      void compute_Vs_anomaly(Vector<float> &values) const;

      void compute_Vp_anomaly(Vector<float> &values) const;

      double normalize_pressure(LinearAlgebra::BlockVector &vector) const;

      void denormalize_pressure(const double                      pressure_adjustment,
                                LinearAlgebra::BlockVector       &vector,
                                const LinearAlgebra::BlockVector &relevant_vector) const;

      void apply_limiter_to_dg_solutions (const AdvectionField &advection_field);


      void compute_reactions ();

      void update_solution_vectors_with_reaction_results (const unsigned int block_index,
                                                          const LinearAlgebra::BlockVector &distributed_vector,
                                                          const LinearAlgebra::BlockVector &distributed_reaction_vector);

      void initialize_current_linearization_point ();

      void interpolate_material_output_into_advection_field (const AdvectionField &adv_field);


      void interpolate_onto_velocity_system(const TensorFunction<1,dim> &func,
                                            LinearAlgebra::Vector &vec);


      void setup_nullspace_constraints(AffineConstraints<double> &constraints);


      void remove_nullspace(LinearAlgebra::BlockVector &relevant_dst,
                            LinearAlgebra::BlockVector &tmp_distributed_stokes);

      RotationProperties<dim>
      compute_net_angular_momentum(const bool use_constant_density,
                                   const LinearAlgebra::BlockVector &solution,
                                   const bool limit_to_top_faces = false) const;

      void remove_net_angular_momentum( const bool use_constant_density,
                                        LinearAlgebra::BlockVector &relevant_dst,
                                        LinearAlgebra::BlockVector &tmp_distributed_stokes,
                                        const bool limit_to_top_faces = false);

      void replace_outflow_boundary_ids(const unsigned int boundary_id_offset);

      void restore_outflow_boundary_ids(const unsigned int boundary_id_offset);

      void remove_net_linear_momentum( const bool use_constant_density,
                                       LinearAlgebra::BlockVector &relevant_dst,
                                       LinearAlgebra::BlockVector &tmp_distributed_stokes);

      double get_maximal_velocity (const LinearAlgebra::BlockVector &solution) const;

      double get_entropy_variation (const double average_field,
                                    const AdvectionField &advection_field) const;

      std::pair<double,double>
      get_extrapolated_advection_field_range (const AdvectionField &advection_field) const;

      void exchange_refinement_flags();


      void maybe_write_timing_output () const;

      bool maybe_write_checkpoint (const time_t last_checkpoint_time,
                                   const bool force_writing_checkpoint);

      bool maybe_do_initial_refinement (const unsigned int max_refinement_level);

      void maybe_refine_mesh (const double new_time_step,
                              unsigned int &max_refinement_level);

      void advance_time (const double step_size);

      double
      compute_viscosity(internal::Assembly::Scratch::AdvectionSystem<dim> &scratch,
                        const double                        global_u_infty,
                        const double                        global_field_variation,
                        const double                        average_field,
                        const double                        global_entropy_variation,
                        const double                        cell_diameter,
                        const AdvectionField     &advection_field) const;

      void
      compute_advection_system_residual(internal::Assembly::Scratch::AdvectionSystem<dim> &scratch,
                                        const double                        average_field,
                                        const AdvectionField               &advection_field,
                                        double                             &max_residual,
                                        double                             &max_velocity,
                                        double                             &max_density,
                                        double                             &max_specific_heat,
                                        double                             &conductivity) const;

      bool
      stokes_matrix_depends_on_solution () const;

      void
      check_consistency_of_formulation ();

      void
      check_consistency_of_boundary_conditions () const;

      double
      compute_initial_newton_residual (const LinearAlgebra::BlockVector &linearized_stokes_initial_guess);

      double
      compute_Eisenstat_Walker_linear_tolerance(const bool EisenstatWalkerChoiceOne,
                                                const double maximum_linear_stokes_solver_tolerance,
                                                const double linear_stokes_solver_tolerance,
                                                const double stokes_residual,
                                                const double newton_residual,
                                                const double newton_residual_old);

      void output_statistics();

      double
      compute_initial_stokes_residual();

      Parameters<dim>                     parameters;

      std::unique_ptr<MeltHandler<dim>> melt_handler;

      std::unique_ptr<NewtonHandler<dim>> newton_handler;

      SimulatorSignals<dim>               signals;

      const IntermediaryConstructorAction post_signal_creation;

      std::unique_ptr<VolumeOfFluidHandler<dim>> volume_of_fluid_handler;

      Introspection<dim>                  introspection;


      MPI_Comm                            mpi_communicator;

      std::ofstream log_file_stream;

      using TeeDevice = boost::iostreams::tee_device<std::ostream, std::ofstream>;
      using TeeStream = boost::iostreams::stream<TeeDevice>;

      TeeDevice iostream_tee_device;
      TeeStream iostream_tee_stream;

      ConditionalOStream                  pcout;

      TableHandler                        statistics;

      std::size_t                         statistics_last_write_size;
      std::size_t                         statistics_last_hash;

      mutable TimerOutput                 computing_timer;

      Timer wall_timer;

      double total_walltime_until_last_snapshot;

      std::thread                         output_statistics_thread;

      const std::unique_ptr<InitialTopographyModel::Interface<dim>>          initial_topography_model;
      const std::unique_ptr<GeometryModel::Interface<dim>>                   geometry_model;
      const IntermediaryConstructorAction                                    post_geometry_model_creation_action;
      const std::unique_ptr<MaterialModel::Interface<dim>>                   material_model;
      const std::unique_ptr<GravityModel::Interface<dim>>                    gravity_model;
      BoundaryTemperature::Manager<dim>                                      boundary_temperature_manager;
      BoundaryComposition::Manager<dim>                                      boundary_composition_manager;
      const std::unique_ptr<PrescribedStokesSolution::Interface<dim>>        prescribed_stokes_solution;

      std::shared_ptr<InitialTemperature::Manager<dim>>                      initial_temperature_manager;
      std::shared_ptr<InitialComposition::Manager<dim>>                      initial_composition_manager;

      const std::unique_ptr<AdiabaticConditions::Interface<dim>>             adiabatic_conditions;
#ifdef ASPECT_WITH_WORLD_BUILDER

      std::shared_ptr<WorldBuilder::World>                                   world_builder;
#endif
      BoundaryVelocity::Manager<dim>                                         boundary_velocity_manager;
      std::map<types::boundary_id,std::unique_ptr<BoundaryTraction::Interface<dim>>> boundary_traction;
      const std::unique_ptr<BoundaryHeatFlux::Interface<dim>>                boundary_heat_flux;

      std::unique_ptr<Particle::World<dim>> particle_world;

      ::Particles::ParticleHandler<dim> particle_handler_copy;

      double                                                    time;
      double                                                    time_step;
      double                                                    old_time_step;
      unsigned int                                              timestep_number;
      unsigned int                                              pre_refinement_step;
      unsigned int                                              nonlinear_iteration;
      TimeStepping::Manager<dim>                                time_stepping_manager;
      LateralAveraging<dim>                                     lateral_averaging;
      parallel::distributed::Triangulation<dim>                 triangulation;
      double                                                    global_Omega_diameter;
      double                                                    global_volume;

      MeshRefinement::Manager<dim>                              mesh_refinement_manager;
      HeatingModel::Manager<dim>                                heating_model_manager;

      std::unique_ptr<Mapping<dim>>                            mapping;

      const FESystem<dim>                                       finite_element;

      DoFHandler<dim>                                           dof_handler;

      Postprocess::Manager<dim>                                 postprocess_manager;

      AffineConstraints<double>                                          constraints;
      AffineConstraints<double>                                          current_constraints;

      double                                                    last_pressure_normalization_adjustment;

      double                                                    pressure_scaling;

      bool                           do_pressure_rhs_compatibility_modification;

      LinearAlgebra::BlockSparseMatrix                          system_matrix;

      LinearAlgebra::BlockSparseMatrix                          system_preconditioner_matrix;

      LinearAlgebra::BlockVector                                solution;
      LinearAlgebra::BlockVector                                old_solution;
      LinearAlgebra::BlockVector                                old_old_solution;
      LinearAlgebra::BlockVector                                system_rhs;

      LinearAlgebra::BlockVector                                current_linearization_point;

      // only used if is_compressible()
      LinearAlgebra::BlockVector                                pressure_shape_function_integrals;

      // only used if operator split is enabled
      LinearAlgebra::BlockVector                                operator_split_reaction_vector;



      std::unique_ptr<LinearAlgebra::PreconditionAMG>           Amg_preconditioner;
      std::unique_ptr<LinearAlgebra::PreconditionBase>          Mp_preconditioner;

      bool                                                      rebuild_sparsity_and_matrices;
      bool                                                      rebuild_stokes_matrix;
      bool                                                      assemble_newton_stokes_matrix;
      bool                                                      assemble_newton_stokes_system;
      bool                                                      rebuild_stokes_preconditioner;

    private:

      std::unique_ptr<MeshDeformation::MeshDeformationHandler<dim>> mesh_deformation;

      std::unique_ptr<StokesMatrixFreeHandler<dim>> stokes_matrix_free;

      friend class boost::serialization::access;
      friend class SimulatorAccess<dim>;
      friend class MeshDeformation::MeshDeformationHandler<dim>;   // MeshDeformationHandler needs access to the internals of the Simulator
      friend class VolumeOfFluidHandler<dim>; // VolumeOfFluidHandler needs access to the internals of the Simulator
      friend class StokesMatrixFreeHandler<dim>;
      template <int dimension, int velocity_degree>
      friend class StokesMatrixFreeHandlerImplementation;
      friend struct Parameters<dim>;
  };
}


#endif