Skip to content

1538 ib state file per process io #1540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sbryngelson merged 4 commits into from
Jun 5, 2026
+189 −78
Merged

1538 ib state file per process io #1540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eb0d050
Fixed restart data to have a file_per_process path as well
danieljvickers Jun 5, 2026
a6ec4b3
Debugged the ib markers not being written with file_per_process
danieljvickers Jun 5, 2026
c26ad4f
Merge branch 'master' into 1538-ib-state-file-per-process-io
danieljvickers Jun 5, 2026
c64f071
Removed duplicate directory creation
danieljvickers Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 45 additions & 17 deletions src/post_process/m_data_output.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1356,6 +1356,8 @@ contains
integer, dimension(num_procs) :: meshtypes
integer :: i, ios, file_unit
integer :: ierr, nBodies
integer :: r, nlocal, gbl_id
character(len=10) :: t_step_string
real(wp), dimension(:), allocatable :: px, py, pz
real(wp), dimension(:), allocatable :: force_x, force_y, force_z
real(wp), dimension(:), allocatable :: torque_x, torque_y, torque_z
Expand All @@ -1365,15 +1367,6 @@ contains
real(wp), dimension(:), allocatable :: ib_diameter

if (proc_rank == 0) then
! Build path to per-timestep IB state file
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)

inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) then
call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')
end if

nBodies = num_ibs

if (nBodies > 0) then
Expand All @@ -1386,16 +1379,51 @@ contains
allocate (angle_x(nBodies), angle_y(nBodies), angle_z(nBodies))
allocate (ib_diameter(nBodies))

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))
if (file_per_process) then
call s_int_to_str(t_step, t_step_string)
ib_data = 0._wp
do r = 0, num_procs - 1
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat'
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)

do i = 1, nBodies
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state file')
ib_data(i,:) = ib_buf(:)
end do
inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))

read (file_unit, iostat=ios) nlocal
if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc))

do i = 1, nlocal
read (file_unit, iostat=ios) gbl_id
if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc))
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc))
ib_data(gbl_id,:) = ib_buf(:)
end do

close (file_unit)
close (file_unit)
end do
else
! Build path to per-timestep IB state file
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)

inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) call s_mpi_abort('Restart file ' // trim(file_loc) // ' does not exist!')

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Cannot open IB state file: ' // trim(file_loc))

do i = 1, nBodies
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state file')
ib_data(i,:) = ib_buf(:)
end do

close (file_unit)
end if

do i = 1, nBodies
force_x(i) = ib_data(i, 2); force_y(i) = ib_data(i, 3); force_z(i) = ib_data(i, 4)
Expand Down
111 changes: 79 additions & 32 deletions src/simulation/m_data_output.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,10 @@ contains
end if

call MPI_FILE_CLOSE(ifile, ierr)

if (ib) then
call s_write_parallel_ib_data(t_step)
end if
else
if (ib) then
call s_initialize_mpi_data(q_cons_vf, ib_markers)
Expand Down Expand Up @@ -908,52 +912,95 @@ contains
integer(kind=MPI_OFFSET_KIND) :: WP_MOK
integer :: ifile, ierr
integer, dimension(MPI_STATUS_SIZE) :: status
logical :: file_exist
logical :: file_exist, dir_check
integer :: i, ib_idx
integer, parameter :: NFIELDS_PER_IB = 20
real(wp) :: ib_buf(NFIELDS_PER_IB)
integer :: file_unit
character(len=10) :: t_step_string

! Partition IBs across ranks round-robin style
integer :: ib_start, ib_end, nibs_per_rank, remainder

WP_MOK = int(storage_size(0._wp)/8, MPI_OFFSET_KIND)

if (proc_rank == 0) then
call s_create_directory(trim(case_dir) // '/restart_data')
end if
call s_mpi_barrier()
if (file_per_process) then
call s_int_to_str(t_step, t_step_string)

write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)
if (proc_rank == 0) then
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string)
call s_create_directory(trim(file_loc))
end if
call s_mpi_barrier()
call DelayFileAccess(proc_rank)

inquire (FILE=trim(file_loc), EXIST=file_exist)
if (file_exist .and. proc_rank == 0) then
call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr)
end if
call s_mpi_barrier()
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', proc_rank, '.dat'
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)

call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)
inquire (FILE=trim(file_loc), EXIST=file_exist)
if (file_exist) then
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='replace')
else
open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='new')
end if

do i = 1, num_local_ibs
ib_idx = local_ib_patch_ids(i)
ib_buf(1) = mytime
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
ib_buf(17) = patch_ib(ib_idx)%x_centroid
ib_buf(18) = patch_ib(ib_idx)%y_centroid
ib_buf(19) = patch_ib(ib_idx)%z_centroid
ib_buf(20) = patch_ib(ib_idx)%radius

! Global IB index (i) determines position in file
disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK

call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr)
end do
write (file_unit) num_local_ibs
do i = 1, num_local_ibs
ib_idx = local_ib_patch_ids(i)
ib_buf(1) = mytime
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
ib_buf(17) = patch_ib(ib_idx)%x_centroid
ib_buf(18) = patch_ib(ib_idx)%y_centroid
ib_buf(19) = patch_ib(ib_idx)%z_centroid
ib_buf(20) = patch_ib(ib_idx)%radius

write (file_unit) patch_ib(ib_idx)%gbl_patch_id
write (file_unit) ib_buf
end do

call MPI_FILE_CLOSE(ifile, ierr)
close (file_unit)
else
if (proc_rank == 0) then
call s_create_directory(trim(case_dir) // '/restart_data')
end if
call s_mpi_barrier()

write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)

inquire (FILE=trim(file_loc), EXIST=file_exist)
if (file_exist .and. proc_rank == 0) then
call MPI_FILE_DELETE(file_loc, mpi_info_int, ierr)
end if
call s_mpi_barrier()

call MPI_FILE_OPEN(MPI_COMM_WORLD, file_loc, ior(MPI_MODE_WRONLY, MPI_MODE_CREATE), mpi_info_int, ifile, ierr)

do i = 1, num_local_ibs
ib_idx = local_ib_patch_ids(i)
ib_buf(1) = mytime
ib_buf(2:4) = patch_ib(ib_idx)%force(1:3)
ib_buf(5:7) = patch_ib(ib_idx)%torque(1:3)
ib_buf(8:10) = patch_ib(ib_idx)%vel(1:3)
ib_buf(11:13) = patch_ib(ib_idx)%angular_vel(1:3)
ib_buf(14:16) = patch_ib(ib_idx)%angles(1:3)
ib_buf(17) = patch_ib(ib_idx)%x_centroid
ib_buf(18) = patch_ib(ib_idx)%y_centroid
ib_buf(19) = patch_ib(ib_idx)%z_centroid
ib_buf(20) = patch_ib(ib_idx)%radius

! Global IB index determines position in file
disp = int(patch_ib(ib_idx)%gbl_patch_id - 1, MPI_OFFSET_KIND)*int(NFIELDS_PER_IB, MPI_OFFSET_KIND)*WP_MOK

call MPI_FILE_WRITE_AT(ifile, disp, ib_buf, NFIELDS_PER_IB, mpi_p, status, ierr)
end do

call MPI_FILE_CLOSE(ifile, ierr)
end if
#endif

end subroutine s_write_parallel_ib_state
Expand Down
94 changes: 65 additions & 29 deletions src/simulation/m_start_up.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1114,47 +1114,83 @@ contains
integer, intent(in) :: t_step
character(len=path_len + 2*name_len) :: file_loc
integer :: i, ios, file_unit, ierr
integer :: r, nlocal, gbl_id
integer, parameter :: NFIELDS_PER_IB = 20
real(wp) :: ib_buf(NFIELDS_PER_IB)
logical :: file_exist
character(len=10) :: t_step_string

write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)
if (file_per_process) then
call s_int_to_str(t_step, t_step_string)

if (proc_rank == 0) then
inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) then
call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))
end if
do r = 0, num_procs - 1
write (file_loc, '(A,I0,A,i7.7,A)') 'ib_state_', t_step, '_', r, '.dat'
file_loc = trim(case_dir) // '/restart_data/lustre_' // trim(t_step_string) // '/' // trim(file_loc)

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))
inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))

read (file_unit, iostat=ios) nlocal
if (ios /= 0) call s_mpi_abort('Error reading IB state file header: ' // trim(file_loc))

do i = 1, nlocal
read (file_unit, iostat=ios) gbl_id
if (ios /= 0) call s_mpi_abort('Error reading IB patch ID: ' // trim(file_loc))
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state data: ' // trim(file_loc))

patch_ib(gbl_id)%vel = ib_buf(8:10)
patch_ib(gbl_id)%angular_vel = ib_buf(11:13)
patch_ib(gbl_id)%angles = ib_buf(14:16)
patch_ib(gbl_id)%x_centroid = ib_buf(17)
patch_ib(gbl_id)%y_centroid = ib_buf(18)
patch_ib(gbl_id)%z_centroid = ib_buf(19)
end do

do i = 1, num_ibs
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state restart file')

patch_ib(i)%vel = ib_buf(8:10)
patch_ib(i)%angular_vel = ib_buf(11:13)
patch_ib(i)%angles = ib_buf(14:16)
patch_ib(i)%x_centroid = ib_buf(17)
patch_ib(i)%y_centroid = ib_buf(18)
patch_ib(i)%z_centroid = ib_buf(19)
close (file_unit)
end do
else
write (file_loc, '(A,I0,A)') '/restart_data/ib_state_', t_step, '.dat'
file_loc = trim(case_dir) // trim(file_loc)

close (file_unit)
end if
if (proc_rank == 0) then
inquire (FILE=trim(file_loc), EXIST=file_exist)
if (.not. file_exist) then
call s_mpi_abort('Cannot open IB state file for restart: ' // trim(file_loc))
end if

open (newunit=file_unit, file=trim(file_loc), form='unformatted', access='stream', status='old', iostat=ios)
if (ios /= 0) call s_mpi_abort('Error opening IB state restart file: ' // trim(file_loc))

do i = 1, num_ibs
read (file_unit, iostat=ios) ib_buf
if (ios /= 0) call s_mpi_abort('Error reading IB state restart file')

patch_ib(i)%vel = ib_buf(8:10)
patch_ib(i)%angular_vel = ib_buf(11:13)
patch_ib(i)%angles = ib_buf(14:16)
patch_ib(i)%x_centroid = ib_buf(17)
patch_ib(i)%y_centroid = ib_buf(18)
patch_ib(i)%z_centroid = ib_buf(19)
end do

close (file_unit)
end if

#ifdef MFC_MPI
do i = 1, num_ibs
call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
end do
do i = 1, num_ibs
call MPI_BCAST(patch_ib(i)%vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%angular_vel, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%angles, 3, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%x_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%y_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(patch_ib(i)%z_centroid, 1, mpi_p, 0, MPI_COMM_WORLD, ierr)
end do
#endif
end if

end subroutine s_read_ib_restart_data

Expand Down
Loading