Include new fortran90 module into existing huge fortran90 model

I am a beginner of fortran90. What I want to do is to call a new module (also fortran90) in an existing huge model which was written by fortran90. What I knew is that running fortran scripts highly depends on the compilation by gfortran using make or cmake. And make&cmake will compile the Makefile&CMakefile. I checked that, this huge model uses commands make with a large Makefile. The new module I wanted to use contains several .f90 files. So it’s impossible to just simply write this module into one existing .f90 file of the model…And one thing is that, I just want to use this new module in ONE subroutine in the existing fortran90 model.

For the new module, it’s downloaded from GitHub and the compilation was done by cmake. Firstly I need to run a bash file !sh build_steps.sh (the ! is used for running bash commands in python terminal). And the !sh build_steps.sh simply follows:

rm -rf build
mkdir build
cd build

FC=gfortran cmake .. -DSERIAL=1
# FC='mpif90 -qopenmp' cmake .. -DSERIAL=1

make

cd CMakeFiles/neural.dir/

mv mod_activation.mod.stamp mod_activation.o
mv mod_io.mod.stamp mod_io.o
mv mod_kinds.mod.stamp mod_kinds.o
mv mod_layer.mod.stamp mod_layer.o
mv mod_mnist.mod.stamp mod_mnist.o
mv mod_network.mod.stamp mod_network.o
mv mod_parallel.mod.stamp mod_parallel.o
mv mod_random.mod.stamp mod_random.o
mv mod_ensemble.mod.stamp mod_ensemble.o
mv mod_dense_layer.mod.stamp mod_dense_layer.o
mv mod_batchnorm_layer.mod.stamp mod_batchnorm_layer.o
mv mod_dropout_layer.mod.stamp mod_dropout_layer.o

Then the CMakelists.txt follows:

# cmake version, project name, language
cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
project(neural-fortran Fortran)

# set output paths for modules, archives, and executables
set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/include)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

# if build type not specified, default to release
if(NOT CMAKE_BUILD_TYPE)
  set(CMAKE_BUILD_TYPE "release")
endif()

# handle integer size
if(INT)
  message(STATUS "Configuring build for ${INT}-bit integers")
  add_definitions(-DINT${INT})
else()
  message(STATUS "Configuring build for 32-bit integers")
  add_definitions(-DINT32)
endif()

# handle real size
if(REAL)
  message(STATUS "Configuring build for ${REAL}-bit reals")
  add_definitions(-DREAL${REAL})
else()
  message(STATUS "Configuring build for 32-bit reals")
  add_definitions(-DREAL32)
endif()

if(SERIAL)
  message(STATUS "Configuring build for serial execution")
else()
  message(STATUS "Configuring build for parallel execution")
  add_definitions(-DCAF)
endif()

# compiler flags for gfortran
if(CMAKE_Fortran_COMPILER_ID MATCHES GNU)

  if(SERIAL)
    message(STATUS "Configuring to build with -fcoarray=single")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fcoarray=single")
  endif()

  if(BLAS)
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fexternal-blas ${BLAS}")
    set(LIBS "${LIBS} blas")
    message(STATUS "Configuring build to use BLAS from ${BLAS}")
  endif()

  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp -fopenmp")
  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -C -fbacktrace")
  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -ffast-math")
endif()

# compiler flags for ifort
if(CMAKE_Fortran_COMPILER_ID MATCHES Intel)

  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fpp -assume byterecl,realloc_lhs -heap-arrays -qopenmp")
  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -C -traceback")
  set(CMAKE_Fortran_FLAGS_RELEASE "-O3")

  if(NOT SERIAL)
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -coarray=shared")
  endif()

endif()

# compiler flags for Cray ftn
if(CMAKE_Fortran_COMPILER_ID MATCHES Cray)
  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -h noomp")
  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g")
  set(CMAKE_Fortran_FLAGS_RELEASE "-O3")
endif()

# library to archive (libneural.a)
add_library(neural src/lib/mod_activation.F90 src/lib/mod_io.F90 src/lib/mod_kinds.F90 src/lib/mod_layer.F90 src/lib/mod_dense_layer.F90 src/lib/mod_dropout_layer.F90 src/lib/mod_batchnorm_layer.F90 src/lib/mod_mnist.F90 src/lib/mod_network.F90 src/lib/mod_ensemble.F90 src/lib/mod_parallel.F90 src/lib/mod_random.F90)

# Remove leading or trailing whitespace
string(REGEX REPLACE "^ | $" "" LIBS "${LIBS}")

# tests
enable_testing()
#  mnist network_save network_sync set_activation_function
foreach(execid keras bulk ensembles training save_and_load keras_mymodel)
  add_executable(test_${execid} src/tests/test_${execid}.F90)
  target_link_libraries(test_${execid} neural ${LIBS})
  add_test(test_${execid} bin/test_${execid})
endforeach()

# foreach(execid mnist save_and_load simple sine)
#   add_executable(example_${execid} src/tests/example_${execid}.F90)
#   target_link_libraries(example_${execid} neural ${LIBS})
#   add_test(example_${execid} bin/example_${execid})
# endforeach()

The Makefile is too long, so I keep it in a google doc Makefile - Google Docs. (this model is open-source…)

I knew something about bash, but nothing about Make and Cmake. So I just want to know, is there a simple way to combine this two makefiles? how can I declare the subroutine dependency in the existing makefile? or just simply import the new module in the existing fortran 90 subroutine like import numpy in python. Or do I need to change the dependency one by one?

Thanks a lot!

Cross-posted to makefile - Include new fortran90 module into existing huge fortran90 model - Stack Overflow Some discussion is already present there.

1 Like

Welcome @xushanthu to the Fortran Discourse!
The author of neural-fortran, @milancurcic , can probably help you about how using its library in another program.

I note that there is some terminological confusion with the word module. It means “a bunch of code” and “a Fortran MODULE”.

The new code you want to call from your code-base will have its own build procedure. The end result of this build should be an object-code library and possibly some .mod files. Are you at that point where you can produce those?

Yes…I was the author there…and when I was just waiting for responses from people from stackoverflow, I found this community so I put it here for more people…

Yes, you are right…I can run the test_keras.f90 from neural-fortran to play with it…but just simply does not know how to transfer it to my original model…

Hi @themos, thanks for your reply! Yes, the module means “a Fortran MODULE”, and also this module could involve a bunch of .f90 codes.
Yes, I could play with the new module with itself. And I found its compilation makes a new directory “build”. And the file structure is like follows:


So I suppose the “./build/include/mod_activation.mod” and or so is the “.mod files”? and also the object-code library is the “./build/lib/libneural.a”? So I think I at that point and I can produce .mod files and library.

You can certainly ask here. I would even argue that it will be hard to answer it in the Stack Overflow format because it is too unclear or contains too many points and some misconceptions for that.

For example you write:

Yes, the module means “a Fortran MODULE”, and also this module could involve a bunch of .f90 codes.

I have no idea what that might mean. Does the new module just use other modules? Or does it use submodules (a Fortran 2008 feature)? Or is it actually a program that will use the neural networks library?

Hi @VladimirF, thanks for your reply! Yes, in this new module, the fortran90 subroutines are involved with each other files under the same directory. For example (see the file structure above). In test_keras.F90, I want to use mod_network as

program test_keras
  use mod_kinds, only: ik, rk
  use mod_network, only: network_type

  implicit none

  type(network_type) :: net

  real(rk), allocatable :: result1(:), input(:), input_1(:,:)
  character(len=100), dimension(:), allocatable :: args

  allocate(args(1))
  call get_command_argument(1,args(1))

  ! load trained network from keras
  call net % load(args(1))

  input = [1, 2, 3, 4, 5]
  result1 = net % output(input)
  print *, result1

end program test_keras

Then Let’s see what is stated in ./lib/mod_network.f90, in the beginning of ./lib/mod_network.f90, a lot of other modules (also in this new package) are used…it’s like:

module mod_network

  use mod_kinds, only: ik, rk
  use mod_layer, only: array1d, array2d, db_init, dw_init,&
                       db_co_sum, dw_co_sum, layer_type
  use mod_dense_layer, only: Dense
  use mod_dropout_layer, only: Dropout
  use mod_batchnorm_layer, only: BatchNorm
  use mod_parallel, only: tile_indices
...
end module mod_network

So as you see, this file is interacted with many files, such as mod_kinds.F90 and other F90 files (see it in the screenshot in my reply to @themos )…some people suggest me to add all of this in one subroutine in my atmospheric model…but I don’t think it’s a good idea. Those .f90 files are highly interactive with each other, that’s why I posted its own CMakeLists.txt here.

And in my atmospheric model, for example, in one subroutine, the original script would be:

SUBROUTINE whatever(arg1, arg2, xxx)

USE MOD ATMOS_1
USE MOD ATMOS_2
...
END SUBROUTINE whatever

And I just want to add one line use mod_network, only: network_type into the whatever.f90 subroutine…but adding mod_network does not mean I just only add one mod_network.mod, I need to add other .mod files and also state their dependency…

From the discussions so far it sounds like you want to add a whole library to your existing project. (library - a collection of Fortran modules that work together as (mostly) standalone unit, possibly depending on other libraries; project - Fortran program(s) to be used for some purpose, possibly built out of more modules and libraries).

First some background. In order to compile a Fortran source file, the compiler must be able to find all of the *.mod files for all of the modules used by that source file. The compilation process will produce a *.mod file for each module contained in the source file. This is how one can determine the dependencies (either manually or using some tool) to figure out the order in which the files must be compiled. So you need to tell the compiler where to find the *.mod files for your new library. When linking (to produce an executable) you must provide the linker (often called as the same program as the compiler) with all of the object files that contain all of the code used by the program. In this case the build system of the library you wish to add has been nice enough to combine all of its object files together into a single archive (the *.a file), so you only need to include that one in the link command.

Combining packages that use different build systems is not easy to say the least. It’s doable, and there are some shortcuts you could take to try things out, but unless you want your project’s build system to turn into a cobbled together nightmare that requires lots of manual intervention for every build, I suggest spending the effort to really do it right. If you can, try and switch everything you need to using fpm. I’ve been doing this for all my projects and it saves so much time in the long run.

Good luck, and if you need help, I do offer my services under contract work through Sourcery Inc, so don’t hesitate to reach out.

Hi @everythingfunctional, thanks a lot for your detailed explanation! Could you please explain “so you only need to include that one in the link command”?
Does that mean, I need to copy libneural.a into the library directory of my existing project, and add one line like gfortran using_FKB.f90 -L/path_of_llib/ -llibnerual.a in the Makefile of my existing project?
Thanks a lot!

On the command that creates the executable (i.e. gfortran -o using_FKB using_FKB.o other.o ...) you would either add something like -L/path_of_lib -lneural or path/to/libneural.a.

1 Like

Thanks @everythingfunctional! I will try this and see whether it works out!

Hi @everythingfunctional, I just found that, in the installation and building process, it requires ./configure . So what else do I need to do beyond just "adding one line like -L/path_of_lib -lneural or path/to/libneural.a "? Thanks a lot!

Installation and building process for which thing?

For the existing fortran project

For your own use, just use configure+make normally and when a command (probably the link stage of creating executables) fails because the new functions are not found, issue a corrected command (with the “-L … -l …” as above) directly from the shell.
If you have added USE statements with the new modules in the source, you will have to correct the compilation lines in a similar fashion.
Some configure scripts allow you to use environment variables that specify additional compilation and linking options. These tend to be called FCFLAGS and LDFLAGS (the configure script should specify the names). You can try setting those and see if configure uses them.

I take it your project is using autotools, and you want to know what else needs to be included with the installation. I’ve never set up an autotools project, so I’m not sure exactly how to do it, but there are two ways to go about it in this instance. Either install the new library separately and tell autotools that it needs to be present and where to find it, or tell autotools that the archive and .mod files must also be installed along with your project.

Again, if you want more detailed and professional help, I do contract work specializing in Fortran and setting up build, test, and CI systems. Don’t hesitate to let me know if you’re interested.

Hi @themos, thanks for your reply! I didn’t log into this website…sorry for the late response! But how should I use FCFLAGS and LDFALGS along with ./configure? something like:

export FCFLAGS="xxxx"
./configure

?
or something else? Thanks!

I would try that first.