diff --git a/src/post_process/m_data_output.fpp b/src/post_process/m_data_output.fpp index 8f0fe445e6..72087dee1b 100644 --- a/src/post_process/m_data_output.fpp +++ b/src/post_process/m_data_output.fpp @@ -1356,6 +1356,8 @@ contains integer, dimension(num_procs) :: meshtypes integer :: i, ios, file_unit integer :: ierr, nBodies + integer :: r, nlocal, gbl_id + character(len=10) :: t_step_string real(wp), dimension(:), allocatable :: px, py, pz real(wp), dimension(:), allocatable :: force_x, force_y, force_z real(wp), dimension(:), allocatable :: torque_x, torque_y, torque_z @@ -1365,15 +1367,6 @@ contains real(wp), dimension(:), allocatable :: ib_diameter if (proc_rank == 0) then - ! Build path to per-timestep IB state file - write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' - file_loc = trim(case_dir) // trim(file_loc) - - inquire (FILE=trim(file_loc), EXIST=file_exist) - if (.not. file_exist) then - call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!') - end if - nBodies = num_ibs if (nBodies > 0) then @@ -1386,16 +1379,51 @@ contains allocate (angle_x(nBodies), angle_y(nBodies), angle_z(nBodies)) allocate (ib_diameter(nBodies)) - open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) - if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc)) + if (file_per_process) then + call s_int_to_str(t_step, t_step_string) + ib_data = 0._wp + do r = 0, num_procs - 1 + write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat' + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc) - do i = 1, nBodies - read (file_unit, iostat=ios) ib_buf - if (ios /= 0) call s_mpi_abort('Error reading IB state file') - ib_data(i,:) = ib_buf(:) - end do + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!') + + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) + if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc)) + + read (file_unit, iostat=ios) nlocal + if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc)) + + do i = 1, nlocal + read (file_unit, iostat=ios) gbl_id + if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc)) + read (file_unit, iostat=ios) ib_buf + if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc)) + ib_data(gbl_id,:) = ib_buf(:) + end do - close (file_unit) + close (file_unit) + end do + else + ! Build path to per-timestep IB state file + write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' + file_loc = trim(case_dir) // trim(file_loc) + + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!') + + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) + if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc)) + + do i = 1, nBodies + read (file_unit, iostat=ios) ib_buf + if (ios /= 0) call s_mpi_abort('Error reading IB state file') + ib_data(i,:) = ib_buf(:) + end do + + close (file_unit) + end if do i = 1, nBodies force_x(i) = ib_data(i, 2); force_y(i) = ib_data(i, 3); force_z(i) = ib_data(i, 4) diff --git a/src/simulation/m_data_output.fpp b/src/simulation/m_data_output.fpp index a4099937b6..6c6935cefc 100644 --- a/src/simulation/m_data_output.fpp +++ b/src/simulation/m_data_output.fpp @@ -753,6 +753,10 @@ contains end if call MPI_FILE_CLOSE(ifile, ierr) + + if (ib) then + call s_write_parallel_ib_data(t_step) + end if else if (ib) then call s_initialize_mpi_data(q_cons_vf, ib_markers) @@ -908,52 +912,95 @@ contains integer(kind=MPI_OFFSET_KIND) :: WP_MOK integer :: ifile, ierr integer, dimension(MPI_STATUS_SIZE) :: status - logical :: file_exist + logical :: file_exist, dir_check integer :: i, ib_idx integer, parameter :: NFIELDS_PER_IB = 20 real(wp) :: ib_buf(NFIELDS_PER_IB) + integer :: file_unit + character(len=10) :: t_step_string ! Partition IBs across ranks round-robin style integer :: ib_start, ib_end, nibs_per_rank, remainder WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND) - if (proc_rank == 0) then - call s_create_directory(trim(case_dir) // '/restart_data') - end if - call s_mpi_barrier() + if (file_per_process) then + call s_int_to_str(t_step, t_step_string) - write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' - file_loc = trim(case_dir) // trim(file_loc) + if (proc_rank == 0) then + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) + call s_create_directory(trim(file_loc)) + end if + call s_mpi_barrier() + call DelayFileAccess(proc_rank) - inquire (FILE=trim(file_loc), EXIST=file_exist) - if (file_exist .and. proc_rank == 0) then - call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr) - end if - call s_mpi_barrier() + write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', proc_rank, '.dat' + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc) - call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr) + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (file_exist) then + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='replace') + else + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='new') + end if - do i = 1, num_local_ibs - ib_idx = local_ib_patch_ids(i) - ib_buf(1) = mytime - ib_buf(2:4) = patch_ib(ib_idx)%force(1:3) - ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3) - ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3) - ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3) - ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3) - ib_buf(17) = patch_ib(ib_idx)%x_centroid - ib_buf(18) = patch_ib(ib_idx)%y_centroid - ib_buf(19) = patch_ib(ib_idx)%z_centroid - ib_buf(20) = patch_ib(ib_idx)%radius - - ! Global IB index (i) determines position in file - disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK - - call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr) - end do + write (file_unit) num_local_ibs + do i = 1, num_local_ibs + ib_idx = local_ib_patch_ids(i) + ib_buf(1) = mytime + ib_buf(2:4) = patch_ib(ib_idx)%force(1:3) + ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3) + ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3) + ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3) + ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3) + ib_buf(17) = patch_ib(ib_idx)%x_centroid + ib_buf(18) = patch_ib(ib_idx)%y_centroid + ib_buf(19) = patch_ib(ib_idx)%z_centroid + ib_buf(20) = patch_ib(ib_idx)%radius + + write (file_unit) patch_ib(ib_idx)%gbl_patch_id + write (file_unit) ib_buf + end do - call MPI_FILE_CLOSE(ifile, ierr) + close (file_unit) + else + if (proc_rank == 0) then + call s_create_directory(trim(case_dir) // '/restart_data') + end if + call s_mpi_barrier() + + write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' + file_loc = trim(case_dir) // trim(file_loc) + + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (file_exist .and. proc_rank == 0) then + call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr) + end if + call s_mpi_barrier() + + call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr) + + do i = 1, num_local_ibs + ib_idx = local_ib_patch_ids(i) + ib_buf(1) = mytime + ib_buf(2:4) = patch_ib(ib_idx)%force(1:3) + ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3) + ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3) + ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3) + ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3) + ib_buf(17) = patch_ib(ib_idx)%x_centroid + ib_buf(18) = patch_ib(ib_idx)%y_centroid + ib_buf(19) = patch_ib(ib_idx)%z_centroid + ib_buf(20) = patch_ib(ib_idx)%radius + + ! Global IB index determines position in file + disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK + + call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr) + end do + + call MPI_FILE_CLOSE(ifile, ierr) + end if #endif end subroutine s_write_parallel_ib_state diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 7de68ea9b7..8114ad4d2f 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -1114,47 +1114,83 @@ contains integer, intent(in) :: t_step character(len=path_len + 2*name_len) :: file_loc integer :: i, ios, file_unit, ierr + integer :: r, nlocal, gbl_id integer, parameter :: NFIELDS_PER_IB = 20 real(wp) :: ib_buf(NFIELDS_PER_IB) logical :: file_exist + character(len=10) :: t_step_string - write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' - file_loc = trim(case_dir) // trim(file_loc) + if (file_per_process) then + call s_int_to_str(t_step, t_step_string) - if (proc_rank == 0) then - inquire (FILE=trim(file_loc), EXIST=file_exist) - if (.not. file_exist) then - call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc)) - end if + do r = 0, num_procs - 1 + write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat' + file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc) - open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) - if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc)) + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (.not. file_exist) call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc)) + + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) + if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc)) + + read (file_unit, iostat=ios) nlocal + if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc)) + + do i = 1, nlocal + read (file_unit, iostat=ios) gbl_id + if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc)) + read (file_unit, iostat=ios) ib_buf + if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc)) + + patch_ib(gbl_id)%vel = ib_buf(8:10) + patch_ib(gbl_id)%angular_vel = ib_buf(11:13) + patch_ib(gbl_id)%angles = ib_buf(14:16) + patch_ib(gbl_id)%x_centroid = ib_buf(17) + patch_ib(gbl_id)%y_centroid = ib_buf(18) + patch_ib(gbl_id)%z_centroid = ib_buf(19) + end do - do i = 1, num_ibs - read (file_unit, iostat=ios) ib_buf - if (ios /= 0) call s_mpi_abort('Error reading IB state restart file') - - patch_ib(i)%vel = ib_buf(8:10) - patch_ib(i)%angular_vel = ib_buf(11:13) - patch_ib(i)%angles = ib_buf(14:16) - patch_ib(i)%x_centroid = ib_buf(17) - patch_ib(i)%y_centroid = ib_buf(18) - patch_ib(i)%z_centroid = ib_buf(19) + close (file_unit) end do + else + write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat' + file_loc = trim(case_dir) // trim(file_loc) - close (file_unit) - end if + if (proc_rank == 0) then + inquire (FILE=trim(file_loc), EXIST=file_exist) + if (.not. file_exist) then + call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc)) + end if + + open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios) + if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc)) + + do i = 1, num_ibs + read (file_unit, iostat=ios) ib_buf + if (ios /= 0) call s_mpi_abort('Error reading IB state restart file') + + patch_ib(i)%vel = ib_buf(8:10) + patch_ib(i)%angular_vel = ib_buf(11:13) + patch_ib(i)%angles = ib_buf(14:16) + patch_ib(i)%x_centroid = ib_buf(17) + patch_ib(i)%y_centroid = ib_buf(18) + patch_ib(i)%z_centroid = ib_buf(19) + end do + + close (file_unit) + end if #ifdef MFC_MPI - do i = 1, num_ibs - call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) - call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) - end do + do i = 1, num_ibs + call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr) + end do #endif + end if end subroutine s_read_ib_restart_data