""":code:`Pymake()` class to make a binary executable for a FORTRAN, C, or C++
program, such as MODFLOW 6.
An example of how to build MODFLOW-2005 from source files in the official
release downloaded from the USGS using Intel compilers is:
.. code-block:: python
import pymake
# create an instance of the Pymake object
pm = pymake.Pymake(verbose=True)
# reset select pymake settings
pm.target = "mf2005"
pm.appdir = "../bin"
pm.fc = "ifort"
pm.cc = "icc"
pm.fflags = "-O3 -fbacktrace"
pm.cflags = "-O3"
# download the target
pm.download_target(pm.target, download_path="temp")
# build the target
pm.build()
# clean up downloaded files
pm.finalize()
All other settings not specified in the script would be based on command
line arguments or default values. The same :code:`Pymake()` object could be
used to compile MODFLOW 6 by appending the following code to the previous code
block:
.. code-block:: python
# reset the target
pm.target = "mf6"
# download the target
pm.download_target(pm.target, download_path="temp")
# build the target
pm.build()
# clean up downloaded files
pm.finalize()
The Intel compilers and fortran flags defined previously would be used when
MODFLOW 6 was built.
"""
import argparse
import os
import shutil
import sys
import time
from pathlib import Path
from zipfile import ZipFile
from .config import __description__
from .pymake_base import main
from .pymake_parser import _get_standard_arg_dict, _parser_setup
from .utils._compiler_switches import (
_get_osname,
)
from .utils._usgs_src_update import _build_replace
from .utils.download import download_and_unzip, zip_all
from .utils.usgsprograms import usgs_program_data
# targets whose distributed meson build files are removed after download, so
# that pymake writes its own. the mp6 build file lists source files by name,
# which the mp6 source update replaces with renamed copies
_IGNORE_PROVIDED_MESON = ("mp6",)
[docs]
class Pymake:
"""Pymake class for interacting with pymake functionality. This is essentially
a wrapper for all of the pymake functions needed to download and build
a target.
"""
def __init__(self, name="pymake", verbose=None):
self.name = name
self.url = None
self.download = None
self.download_path = None
self.download_dir = None
self.returncode = 0
self.build_targets = []
# initialize class variables available as argv items
self.target = None
self.srcdir = None
self.fc = None
self.cc = None
self.makeclean = None
self.double = None
self.debug = None
self.include_subdirs = None
self.fflags = None
self.cflags = None
self.syslibs = None
self.makefile = None
self.makefile_only = None
self.makefiledir = None
self.srcdir2 = None
self.extrafiles = None
self.excludefiles = None
self.sharedobject = None
self.appdir = None
self.keep = None
self.zip = None
self.inplace = None
self.mesondir = None
# the mesondir pymake set, so that one the user set is not replaced
self._set_mesondir_value = None
# set class variables with default values from arg_dict
for key, value in _get_standard_arg_dict().items():
setattr(self, key, value["default"])
# parse command line arguments if python is running script
if sys.argv[0].lower().endswith(".py") or "make-program" in sys.argv[0].lower():
self._arg_parser()
# reset select variables using passed variables
if verbose is not None:
self.verbose = verbose
# reset fortran and c/c++ if fc and cc environmental variables are set
env_var = os.environ.get("FC")
if env_var is not None:
if env_var != self.fc:
self.fc = env_var
env_var = os.environ.get("CC")
if env_var is not None:
if env_var != self.cc:
self.cc = env_var
[docs]
def reset(self, target):
"""Reset PyMake object variables for a target.
Parameters
----------
target : str
target name
Returns
-------
"""
if self.verbose:
print("resetting Pymake class")
self.target = target
self.srcdir = None
[docs]
def finalize(self):
"""Finalize Pymake class.
Returns
-------
"""
if self.download:
self._download_cleanup()
def _print_settings(self):
"""Print settings defined by command line arguments.
Returns
-------
"""
print("\nPymake settings\n" + 30 * "-")
for key, value in _get_standard_arg_dict().items():
print_value = getattr(self, key, value["default"])
if isinstance(print_value, list):
print_value = ", ".join(print_value)
print(f" {key}={print_value}")
print("\n")
[docs]
def argv_reset_settings(self, args):
"""Reset settings using command line arguments.
Parameters
----------
args : Namespace object
reset self.variables using command line arguments
Returns
-------
"""
for key in args.__dict__:
if args.__dict__[key] is not None:
setattr(self, key, args.__dict__[key])
return
def _arg_parser(self):
"""Setup argparse object for Pymake object using only optional
command line arguments.
Returns
-------
"""
loc_dict = _get_standard_arg_dict()
parser = argparse.ArgumentParser(description=__description__)
for _, value in loc_dict.items():
tag = value["tag"][0]
# only process optional command line variables
if tag.startswith("-"):
parser = _parser_setup(parser, value, reset_default=True)
# reset self.variables using optional command line arguments
self.argv_reset_settings(parser.parse_args())
return
[docs]
def compress_targets(self):
"""Compress targets in build_targets list.
Returns
-------
"""
zip_pth = self.zip
if zip_pth is not None:
targets = []
appdir = self.appdir
# list of applications build at this time
if len(self.build_targets) > 0:
for target in self.build_targets:
targets.append(Path(target).name)
# set appdir based on first target, assumes that the path
# for all of the targets are the same
if appdir is None:
appdir = str(Path(target).parent)
# determine files in appdir if no applications build at this
# time (--keep command line argument)
else:
if appdir is None:
appdir = "."
for pth in sorted(Path(appdir).iterdir()):
targets.append(pth.name)
# add code.json
if Path("code.json").exists():
if "code.json" not in targets:
targets.append("code.json")
# delete the zip file if it exists
zip_str = str(Path(zip_pth).resolve())
if Path(zip_pth).exists():
if self.keep:
if self.verbose:
print(f"Appending files to existing zipfile '{zip_str}'")
with ZipFile(zip_pth, mode="r") as zf:
available_file_paths = zf.namelist()
pop_list = []
for target in targets:
for temp_path in available_file_paths:
if target == temp_path:
pop_list.append(target)
break
for target in pop_list:
targets.remove(target)
else:
if self.verbose:
print(f"Deleting existing zipfile '{zip_str}'")
Path(zip_pth).unlink()
# print a message describing the zip process
if self.verbose:
print(
f"Compressing files in '{appdir}' directory to zip file '{zip_str}'"
)
for idx, target in enumerate(targets):
print(f" {idx + 1:>3d}. adding '{target}' to zipfile")
# compress the compiled executables
if not zip_all(
zip_pth, dir_pths=appdir, patterns=targets, append=self.keep
):
self.returncode = 1
return
def _clean_targets(self):
"""Clean up list of targets.
Returns
-------
"""
for target in self.build_targets:
if Path(target).exists():
msg = f"removing '{target}'"
Path(target).unlink()
else:
msg = f"'{target}' does not exist"
if self.verbose:
print(msg)
# reset build_targets
self.build_targets = []
return
[docs]
def download_setup(
self, target, url=None, download_path=".", verify=True, timeout=30
):
"""Setup download.
Parameters
----------
target : str
target name
url : str
url of asset
download_path : str
path where the asset will be saved
verify : bool
boolean defining ssl verification
timeout : int
download timeout in seconds (default is 30)
Returns
-------
"""
# setup program(s) dictionary
prog_dict = usgs_program_data.get_target(target)
# set url
if url is None:
url = prog_dict.url
# determine if the download url has changed
new_url = False
if self.url != url:
new_url = True
if new_url:
# automatic clean up
if self.download:
self._download_cleanup()
# setup new
self.url = url
self.download = False
self.verify = verify
self.timeout = timeout
self.download_path = download_path
self.download_dir = str(Path(download_path) / prog_dict.dirname)
return
[docs]
def download_target(
self, target, url=None, download_path=".", verify=True, timeout=30
):
"""Setup and download url.
Parameters
----------
target : str
target name
url : str
url of asset
download_path : str
path where the asset will be saved
verify : bool
boolean defining ssl verification
timeout : int
download timeout in seconds (default is 30)
Returns
-------
success : bool
boolean flag indicating download success
"""
# setup the download
self.download_setup(
target, url=url, download_path=download_path, verify=verify, timeout=timeout
)
return self.download_url()
[docs]
def download_url(self):
"""Download files from the url.
Returns
-------
success : bool
boolean flag indicating download success
"""
if not self.download:
# write message
msg = f"downloading...'{self.url}'"
print(msg)
# download the url
self.download = download_and_unzip(
self.url,
pth=self.download_path,
verify=self.verify,
timeout=self.timeout,
verbose=self.verbose,
)
if self.download and self._get_base_target() in _IGNORE_PROVIDED_MESON:
self._remove_provided_meson()
return self.download
def _remove_provided_meson(self):
"""Remove the meson build files distributed with a downloaded target.
Returns
-------
"""
if self.download_dir is None:
return
for fpth in Path(self.download_dir).rglob("meson.build"):
if self.verbose:
print(f"removing the meson build file provided...'{fpth}'")
fpth.unlink()
def _download_cleanup(self):
"""Remove the temporary download directory.
Returns
-------
"""
if self.download is not None:
if self.download:
# write process information
msg = f"cleaning temporary files in...'{self.download_dir}'"
print(msg)
# reset self.download
self.download = None
download_dir = Path(self.download_dir)
if download_dir.is_dir():
ntries = 10
for itries in range(ntries):
# wait to delete on windows
if _get_osname() == "win32":
time.sleep(3)
# remove the directory
try:
shutil.rmtree(download_dir)
if self.verbose:
print(f"removing download directory...'{download_dir}'")
break
except:
if self.verbose:
msg = (
f" removal attempt {itries + 1:>2d} "
f"of {ntries:>2d}"
)
print(msg)
# wait prior to returning on windows
if _get_osname() == "win32":
time.sleep(6)
return
def _set_mesondir(self):
"""Set mesondir to the directory a target was downloaded to.
A meson build reads the build file a target provides, where there is
one, and writes a generated build file where there is not, so both
belong with the source. The default is the current directory, which
two builds of different targets would share. The directory is only
changed when mesondir is still the default, so that a mesondir set by
the user is respected.
Returns
-------
"""
# a mesondir pymake set for a target it built before is replaced,
# since that download directory is removed when the target is done,
# and a mesondir the user set is respected
if self.mesondir is not None and self.mesondir != self._set_mesondir_value:
return
# the build file belongs with the source, which is the directory a
# downloaded target was extracted to
self.mesondir = "." if self.download_dir is None else self.download_dir
self._set_mesondir_value = self.mesondir
if self.verbose:
provided = (Path(self.mesondir) / "meson.build").is_file()
action = "using" if provided else "writing"
print(f"{action} the meson build file in...'{self.mesondir}'")
def _set_include_subdirs(self):
"""Determine if sub-directories in the source directory should be
included.
Parameters
----------
Returns
-------
"""
# determine if source subdirectories should be included
if self._get_base_target() in (
"mf6",
"libmf6",
"gridgen",
"mf6beta",
):
self.include_subdirs = True
else:
self.include_subdirs = False
return
[docs]
def set_build_target_bool(self, target=None):
"""Evaluate if the executable exists and if so and the command line
argument --keep is specified then the executable is not built.
Parameters
----------
target : str
target name. If target is None self.target will be used.
(default is None)
Returns
-------
build : bool
boolean indicating if the executable should be built
"""
if target is None:
target = self.target
if self.appdir is not None:
# os.path.dirname rather than Path.parent, because a target
# with no directory has to compare unequal to an appdir of "."
if os.path.dirname(self.target) != self.appdir:
target = str(Path(self.appdir) / Path(target).name)
build_target = True
if Path(target).exists():
if self.keep:
build_target = False
return build_target
def _get_base_target(self):
"""Get base target name without path and extension.
Returns
-------
target : str
target name without path and extension
"""
target = Path(self.target).name
if target.lower().endswith(".exe"):
target = target[:-4]
elif target.lower().endswith(".dll"):
target = target[:-4]
elif target.lower().endswith(".so"):
target = target[:-3]
elif target.lower().endswith(".dylib"):
target = target[:-6]
return target
def _set_srcdir2(self):
"""Set srcdir2 to compile target. Default is None.
Parameters
----------
Returns
-------
"""
if self.srcdir2 is None:
if self._get_base_target() in ("libmf6",):
self.srcdir2 = str(Path(self.download_dir) / "src")
return
def _set_sharedobject(self):
"""Set sharedobject to compile target. Default is None.
Parameters
----------
Returns
-------
"""
if self._get_base_target() in ("libmf6",):
self.sharedobject = True
else:
self.sharedobject = False
# remove any shared compiler options
for flag in (
"-fPIC",
"-shared",
"-dll",
"-dynamiclib",
):
if self.fflags is not None:
self.fflags = self.fflags.replace(flag, "")
if self.cflags is not None:
self.cflags = self.cflags.replace(flag, "")
if self.syslibs is not None:
self.syslibs = self.syslibs.replace(flag, "")
return
def _set_extrafiles(self):
"""Set extrafiles to compile target. Default is None.
Parameters
----------
Returns
-------
"""
extrafiles = self.extrafiles
if extrafiles is None:
if self._get_base_target() in ("zbud6",):
extrafiles = [
"../../../src/Utilities/CharString.f90",
"../../../src/Utilities/ArrayHandlers.f90",
"../../../src/Utilities/ArrayReaders.f90",
"../../../src/Utilities/BlockParser.f90",
"../../../src/Utilities/Budget.f90",
"../../../src/Utilities/Constants.f90",
"../../../src/Utilities/compilerversion.F90",
"../../../src/Utilities/ErrorUtil.f90",
"../../../src/Utilities/FeatureFlags.f90",
"../../../src/Utilities/GeomUtil.f90",
"../../../src/Utilities/MathUtil.f90",
"../../../src/Utilities/InputOutput.f90",
"../../../src/Utilities/kind.f90",
"../../../src/Utilities/LongLineReader.f90",
"../../../src/Utilities/OpenSpec.f90",
"../../../src/Utilities/sort.f90",
"../../../src/Utilities/defmacro.F90",
"../../../src/Utilities/Sim.f90",
"../../../src/Utilities/SimVariables.f90",
"../../../src/Utilities/version.f90",
"../../../src/Utilities/Message.f90",
"../../../src/Utilities/GridFileReader.f90",
"../../../src/Utilities/HashTable.f90",
]
# evaluate extrafiles type
if extrafiles:
srcdir = Path(self.srcdir).absolute()
if isinstance(extrafiles, list):
for idx, value in enumerate(extrafiles):
fpth = str(Path(srcdir) / value)
extrafiles[idx] = os.path.normpath(fpth)
elif isinstance(extrafiles, str):
fpth = str(Path(srcdir) / extrafiles)
extrafiles = os.path.normpath(fpth)
else:
msg = "invalid extrafiles format - must be a list or string"
raise ValueError(msg)
# reset extrafiles
self.extrafiles = extrafiles
return
def _set_excludefiles(self):
"""Set excludefiles to compile target. Default is None.
Parameters
----------
Returns
-------
"""
if self.excludefiles is None:
if self._get_base_target() in ("libmf6",):
self.excludefiles = [str(Path(self.download_dir) / "src" / "mf6.f90")]
return
[docs]
def build(self, target=None, srcdir=None, modify_exe_name=False):
"""Build the target.
Parameters
----------
target : str
target name. If target is None self.target is used.
(default is None)
srcdir : str
path to directory with source files. (default is None)
modify_exe_name : bool
boolean that determines if the target name can be modified to
include precision (dbl) and debugging (d) indicators.
Returns
-------
"""
if target is not None:
self.target = target
self.srcdir = None
if srcdir is not None:
self.srcdir = srcdir
prog_dict = usgs_program_data.get_target(self.target)
if self.srcdir is None:
self.srcdir = str(Path(self.download_dir) / prog_dict.srcdir)
# set mesondir for a downloaded target that provides a meson build file
self._set_mesondir()
# set include_subdirs for known targets
self._set_include_subdirs()
# set srcdir2 for known targets
self._set_srcdir2()
# set extrafiles for known targets
self._set_extrafiles()
# set excludefiles for known targets
self._set_excludefiles()
# set sharedobject for known targets
self._set_sharedobject()
self.target = self.update_target(self.target, modify_target=modify_exe_name)
build_target = self.set_build_target_bool()
if build_target:
# print Pymake() settings
if self.verbose:
self._print_settings()
# download url if it has not been downloaded
if self.download is not None:
self.download_url()
# update source code, if necessary. the source update for the
# MODFLOW 6 targets only removes the files that need an external
# library, and the meson build file MODFLOW 6 provides lists
# those files and excludes them from the build itself, so they
# are only removed when pymake defines the build
# a makefile is written when only a makefile is asked for, so the
# source files it lists are the ones a makefile can build
if self.makefile_only:
self.makefile = True
external_only = self._get_base_target() in ("mf6", "libmf6", "zbud6")
# a makefile is written from the source pymake finds, so the
# files are removed for a makefile whichever build file is used
# the files are only kept for a build file the target provides,
# which has to be there. a generated build file lists the source
# files pymake finds, so it cannot build them either
provided_meson = (
self.mesondir == self.download_dir
and not self.makefile
and (Path(self.mesondir) / "meson.build").is_file()
)
replace_function = _build_replace(self.target)
if external_only and provided_meson:
replace_function = None
if replace_function is not None:
if self.verbose:
msg = f"replacing select source files for {self.target}\n"
print(msg)
# execute select replace function
replace_function(
self.srcdir,
cc=self.cc,
double=self.double,
)
# write message
print(f"compiling...{self.target}")
# build the target
self.returncode = main(
srcdir=self.srcdir,
target=self.target,
fc=self.fc,
cc=self.cc,
makeclean=self.makeclean,
double=self.double,
debug=self.debug,
include_subdirs=self.include_subdirs,
fflags=self.fflags,
cflags=self.cflags,
syslibs=self.syslibs,
makefile=self.makefile,
makefile_only=self.makefile_only,
makefiledir=self.makefiledir,
srcdir2=self.srcdir2,
extrafiles=self.extrafiles,
excludefiles=self.excludefiles,
sharedobject=self.sharedobject,
appdir=self.appdir,
verbose=self.verbose,
inplace=self.inplace,
mesondir=self.mesondir,
)
# issue error if target was not built
if self.returncode != 0:
raise FileNotFoundError(f"could not build {self.target}")
# add target to list of targets
else:
self.update_build_targets()
return self.returncode
[docs]
def update_build_targets(self):
"""Add target to build_targets list if it is not in the list.
Returns
-------
"""
# os.path.abspath rather than Path.absolute, because it removes the
# ".." segments and so two spellings of a target count as one
if os.path.abspath(self.target) not in self.build_targets:
if self.verbose:
print(f"adding {self.target} to build_targets list")
self.build_targets.append(os.path.abspath(self.target))
return
[docs]
def update_target(self, target, modify_target=False):
"""Update target name with executable extension on Windows and
based on pymake settings.
Parameters
----------
target : str
target name
modify_target : bool
boolean indicating if the target name can be modified based
on pymake double and debug settings (default is False)
Returns
-------
target : str
updated target name
"""
# add extension to target on windows or if shared object
if sys.platform.lower() == "win32":
if self.sharedobject:
ext = ".dll"
else:
ext = ".exe"
elif sys.platform.lower() == "darwin":
if self.sharedobject:
ext = ".dylib"
else:
ext = None
else:
if self.sharedobject:
ext = ".so"
else:
ext = None
if ext is not None:
if Path(target).suffix.lower() != ext:
target += ext
# add double and debug to target name
if modify_target:
if self.double:
pth = Path(target)
filename, file_extension = str(pth.with_suffix("")), pth.suffix
if "dbl" not in filename.lower():
target = filename + "dbl" + file_extension
if self.debug:
pth = Path(target)
filename, file_extension = str(pth.with_suffix("")), pth.suffix
if filename.lower()[-1] != "d":
target = filename + "d" + file_extension
return target