Merge pull request #106 from inab/full_circle

Full circle 1.0.0a2

CITATION.cff

@@ -15,7 +15,6 @@ authors:
     given-names: Daniel
     orcid: "https://orcid.org/0000-0001-8152-0398"
 cff-version: 1.2.0
-date-released: "2024-04-24"
 identifiers:
   - description: WfExS-backend release in Zenodo
     type: doi
@@ -28,4 +27,5 @@ message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/inab/WfExS-backend"
 type: software
 title: "WfExS-backend"
-version: 1.0.0a1
+version: 1.0.0a2
+date-released: "2024-07-28"

INSTALL.md

@@ -14,7 +14,7 @@ just using next bash pattern:
 
 ```bash
 # WFEXS_VER can be either a branch, a tag or a commit hash
-WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
 docker build -t inab/wfexs-backend:${WFEXS_VER} \
 --build-arg wfexs_checkout="${WFEXS_VER}" \
 https://raw.githubusercontent.com/inab/WfExS-backend/${WFEXS_VER}/container_recipes/Dockerfile
@@ -25,7 +25,7 @@ a local copy of the recipe, and next command line from the project root will hel
 
 ```bash
 # WFEXS_VER can be either a branch, a tag or a commit hash
-WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
 mkdir WfExS_docker_build
 cd WfExS_docker_build
 curl -O https://raw.githubusercontent.com/inab/WfExS-backend/${WFEXS_VER}/container_recipes/Dockerfile
@@ -46,7 +46,7 @@ just using next bash pattern:
 
 ```bash
 # WFEXS_VER can be either a branch, a tag or a commit hash
-WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
 podman build -t inab/wfexs-backend:${WFEXS_VER} \
 --build-arg wfexs_checkout="${WFEXS_VER}" \
 https://raw.githubusercontent.com/inab/WfExS-backend/${WFEXS_VER}/container_recipes/Dockerfile
@@ -57,7 +57,7 @@ a local copy of the recipe, and next command line from the project root will hel
 
 ```bash
 # WFEXS_VER can be either a branch, a tag or a commit hash
-WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
 mkdir WfExS_podman_build
 cd WfExS_podman_build
 curl -O https://raw.githubusercontent.com/inab/WfExS-backend/${WFEXS_VER}/container_recipes/Dockerfile
@@ -77,7 +77,7 @@ The precondition is having either Apptainer or Singularity properly setup. There
 
   ```bash
   # WFEXS_VER can be either a branch, a tag or a commit hash
-  WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+  WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
   mkdir WfExS_SIF_build
   cd WfExS_SIF_build
   curl -O https://raw.githubusercontent.com/inab/WfExS-backend/${WFEXS_VER}/container_recipes/Singularity.def
@@ -88,7 +88,7 @@ The precondition is having either Apptainer or Singularity properly setup. There
 
   ```bash
   # WFEXS_VER can be either a branch, a tag or a commit hash
-  WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+  WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
   singularity build \
   --build-arg wfexs_checkout="${WFEXS_VER}" \
   wfexs-backend-${WFEXS_VER}.sif container_recipes/Singularity.def
@@ -100,7 +100,7 @@ The precondition is having either Apptainer or Singularity properly setup. There
 
   ```bash
   # Remember to use the correct tag!!!
-  WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+  WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
   singularity build wfexs-${WFEXS_VER}.sif docker-daemon://inab/wfexs-backend:${WFEXS_VER}
   ```
 
@@ -117,7 +117,7 @@ The precondition is having either Apptainer or Singularity properly setup. There
   cd WfExS_SIF_build
 
   # Remember to use the correct tag!!!
-  WFEXS_VER=574fe343c0b59eecd95afbc67894456359ebe649
+  WFEXS_VER=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
 
   # Next command should be used if you used podman to build the local image
   podman save -o wfexs-backend-${WFEXS_VER}.tar inab/wfexs-backend:${WFEXS_VER}
@@ -200,6 +200,10 @@ There are additional software dependencies beyond core ones. Depending on the lo
 
   * [git](https://git-scm.com/) is used to fetch workflows from git repositories.
 
+  * [libmagic.so] dynamic library is needed by [python-magic](https://pypi.org/project/python-magic/) package.
+
+  * [dot] command (from [GraphViz](https://graphviz.org)) is needed to generate a graphical representation of workflows on Workflow Run RO-Crate generation.
+
   * [gocryptfs](https://nuetzlich.net/gocryptfs/) can be used for the feature of secure intermediate results. It has been tested since version v2.0-beta2 ([releases](https://github.com/rfjakob/gocryptfs/releases) provide static binaries).
 
   * [java](https://openjdk.java.net/): Needed to run Nextflow. Supported Java versions go from version 8 to any version below 15 (Nextflow does not support this last one). Both OpenJDK and Sun implementations should work.

container_recipes/.dockerignore

@@ -0,0 +1,4 @@
+*.sif
+*.tar
+*.tar.*
+*.def

container_recipes/.gitignore

@@ -0,0 +1,3 @@
+*.sif
+*.tar
+*.tar.*

container_recipes/Dockerfile

@@ -1,8 +1,17 @@
+FROM	gcc:13 AS entr_build
+ARG	suid_entrypoint_checkout=adf2da44acf4a5feec10b8c73660e8a9b6a4a03b
+WORKDIR	/
+# hadolint ignore=DL3003
+RUN	git clone --filter=blob:none --no-checkout https://github.com/jmfernandez/suid_entrypoint.git && \
+	cd suid_entrypoint && \
+	git checkout "$suid_entrypoint_checkout" && \
+	gcc -static -o suid_entrypoint suid_entrypoint.c
+
 # The default images of python are based on debian
 FROM	python:3.12
 # These arguments help customizing what it is included in the generated image
-ARG	wfexs_checkout=574fe343c0b59eecd95afbc67894456359ebe649
-ARG	apptainer_version=1.3.2
+ARG	wfexs_checkout=8a0a980f1a5e69064d16f89f8ec31973b2eb0c8b
+ARG	apptainer_version=1.3.3
 # JDK version parameters
 ARG	JDK_MAJOR_VER=11
 ARG	JDK_VER=${JDK_MAJOR_VER}.0.11
@@ -16,7 +25,9 @@ ARG	GOCRYPTFS_VER=v2.4.0
 ARG	STATIC_BASH_VER=5.1.004-1.2.2
 # static busybox version
 ARG	BUSYBOX_VER=1.35.0
+COPY	--from=entr_build /suid_entrypoint/suid_entrypoint /suid_entrypoint
 SHELL	["/bin/bash", "-o", "pipefail", "-c"]
+RUN	chmod u+s /suid_entrypoint
 # Install apptainer
 RUN	DPKG_ARCH=$(dpkg --print-architecture) && \
 	wget -nv \
@@ -35,9 +46,9 @@ RUN	wget -nv -O /etc/apt/keyrings/docker.asc https://download.docker.com/linux/d
 	tee /etc/apt/sources.list.d/docker.list > /dev/null && \
 	apt-get update && \
 	apt-get install -y --no-install-recommends docker-ce-cli
-# Install both podman and encfs
+# Install both podman, encfs and graphviz
 # hadolint ignore=DL3008
-RUN	apt-get install -y podman encfs
+RUN	apt-get install -y podman encfs graphviz
 # Install WfExS, trusting the installers in the commit rather than the
 # one in the docker recipe
 WORKDIR	/
@@ -64,4 +75,5 @@ RUN	JDK_MAJOR_VER="${JDK_MAJOR_VER}" \
 	STATIC_BASH_VER="${STATIC_BASH_VER}" \
 	BUSYBOX_VER="${BUSYBOX_VER}" \
 	bash /full-installer.bash
+ENTRYPOINT [ "/suid_entrypoint" ]
 CMD	["bash"]

container_recipes/Singularity.def

@@ -43,9 +43,9 @@ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
 tee /etc/apt/sources.list.d/docker.list > /dev/null && \
 apt-get update && \
 apt-get install -y --no-install-recommends docker-ce-cli
-# Install both podman and encfs
+# Install both podman, encfs and graphviz
 # hadolint ignore=DL3008
-apt-get install -y podman encfs
+apt-get install -y podman encfs graphviz
 # Install WfExS, trusting the installers in the commit rather than the
 # one in the docker recipe
 mkdir -p /

container_recipes/basic-installer.bash

@@ -16,7 +16,7 @@
 # limitations under the License.
 
 # Getting the installation directory
-scriptDir="$(dirname "$0")"
+scriptDir="$(dirname "$(readlink -f "$0")")"
 case "${scriptDir}" in
 	/*)
 		# Path is absolute
@@ -90,7 +90,7 @@ checkInstallGO() {
 	fi
 }
 
-for cmd in python3 pip ; do
+for cmd in python3 ; do
 	set +e
 	type -a "$cmd" 2> /dev/null
 	retval=$?
@@ -101,6 +101,21 @@ for cmd in python3 pip ; do
 	fi
 done
 
+for cmd in dot ; do
+	set +e
+	type -a "$cmd" 2> /dev/null
+	retval=$?
+	set -e
+	if [ "$retval" -ne 0 ] ; then
+		failed=1
+		echo "ERROR: Command $cmd not found in PATH and needed by WfExS runtime"
+	fi
+done
+
+if [ -n "$failed" ] ; then
+	exit 1
+fi
+
 failed=
 for lib in libmagic.so ; do
 	set +e
@@ -109,7 +124,7 @@ for lib in libmagic.so ; do
 	set -e
 	if [ "$retval" -ne 0 ] ; then
 		failed=1
-		echo "ERROR: Library $lib found in ldconfig cache and needed for the installation"
+		echo "ERROR: Library $lib found in ldconfig cache and needed by WfExS runtime"
 	fi
 done
 
@@ -126,7 +141,49 @@ fi
 
 # Is WfExS already installed??? (case of Docker)
 set +eu
-python3 -P -c "import sys"$'\n'"try:"$'\n'"  import wfexs_backend"$'\n'"except:"$'\n'"  sys.exit(1)"$'\n'"sys.exit(0)"
+python3 --help | grep -q '^-P '
+retval=$?
+if [ "$retval" -eq 0 ] ; then
+	python_p_flag="-P"
+	read -r -d "" CHECKWFEXS <<EOF
+import sys
+
+try:
+	import wfexs_backend
+except:
+	sys.exit(1)
+
+sys.exit(0)
+EOF
+	python3 -P -c "$CHECKWFEXS"
+	retval=$?
+else
+	python_p_flag=""
+	read -r -d "" CHECKWFEXS <<EOF
+import sys
+
+try:
+	# Let's remove current directory
+	sys.path.remove("")
+except:
+	pass
+
+try:
+	import os
+	# Let's remove current directory
+	sys.path.remove(os.getcwd())
+except:
+	pass
+
+try:
+	import wfexs_backend
+except:
+	sys.exit(1)
+
+sys.exit(0)
+EOF
+fi
+python3 $python_p_flag -c "$CHECKWFEXS"
 retval=$?
 set -eu
 if [ "$retval" -eq 0 ] ; then
@@ -161,12 +218,45 @@ if [ -z "$envDir" ]; then
 		# Activating the python environment
 		envActivate="${envDir}/bin/activate"
 		source "${envActivate}"
+
+		# Pip should be available
+		for cmd in pip ; do
+			set +e
+			type -a "$cmd" 2> /dev/null
+			retval=$?
+			set -e
+			if [ "$retval" -ne 0 ] ; then
+				failed=1
+				echo "ERROR: Command $cmd not found in PATH and needed for the installation"
+				exit 1
+			fi
+		done
+
 		pip install --require-virtualenv --upgrade pip wheel
 	fi
 
+	# Pip should be available
+	for cmd in pip ; do
+		set +e
+		type -a "$cmd" 2> /dev/null
+		retval=$?
+		set -e
+		if [ "$retval" -ne 0 ] ; then
+			failed=1
+			echo "ERROR: Command $cmd not found in PATH and needed for the installation"
+			exit 1
+		fi
+	done
+
 	# Checking whether the modules were already installed
-	echo "Installing WfExS-backend python dependencies"
-	pip install --require-virtualenv -r "${requirementsFile}"
+	PYVER=$(python -c 'import sys; print("{}.{}".format(sys.version_info.major, sys.version_info.minor))')
+	constraintsFile="$(readlink -f "${scriptDir}"/../constraints-${PYVER}.txt)"
+	echo "Installing WfExS-backend python dependencies (${PYVER})"
+	PIP_INSTALL_PARAMS=( -r "${requirementsFile}" )
+	if [ -f "$constraintsFile" ] ; then
+		PIP_INSTALL_PARAMS+=( -c "${constraintsFile}" )
+	fi
+	pip install --require-virtualenv "${PIP_INSTALL_PARAMS[@]}"
 
 	# Now, should we run something wrapped?
 	if [ $# != 0 ] ; then

container_recipes/full-installer.bash

@@ -28,7 +28,7 @@
 : ${BUSYBOX_VER:=1.35.0}
 
 # Getting the installation directory
-scriptDir="$(dirname "$0")"
+scriptDir="$(dirname "$(readlink -f "$0")")"
 case "${scriptDir}" in
 	/*)
 		# Path is absolute

setup.py

@@ -81,6 +81,7 @@
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Development Status :: 3 - Alpha",
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: OS Independent",

tests/conftest.py

@@ -17,11 +17,17 @@
 # limitations under the License.
 
 import pytest
+import pathlib
 
 from tests.pushers.marks import (
     MARKERS,
 )
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from py.path.local import LocalPath  # type: ignore[import]
+
 
 def pytest_addoption(parser: "pytest.Parser") -> "None":
     for mark_details in MARKERS:
@@ -31,3 +37,8 @@ def pytest_addoption(parser: "pytest.Parser") -> "None":
             action="store",
             help=mark_details.param_description,
         )
+
+
+@pytest.fixture
+def tmppath(tmpdir: "LocalPath") -> "pathlib.Path":
+    return pathlib.Path(tmpdir)

tests/core/data/test-hello-cwl-secure.wfex.stage

@@ -0,0 +1,13 @@
+workflow_id: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello-workflow.cwl
+workflow_config:
+  secure: true
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello.yml
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File

tests/core/data/test-hello-cwl.wfex.stage

@@ -0,0 +1,13 @@
+workflow_id: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello-workflow.cwl
+workflow_config:
+  secure: false
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello.yml
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File