Prepare Python

To prepare a distribution bundle for setting up a custom environment in a cluster on an Agent, build Python and fetch dependencies as described below.

Build Python

To ensure the required non-root, or standalone, installation of Python, meet the prerequisites below and build Python from the source code.

Meet prerequisites

• Install and configure Docker.
• Install bash.
• Make sure your Internet connection is ok.

Build Python from source code

Follow the instruction:

1. Create the build script as shown below:

   Expand for python_builder.sh code

   #!/usr/bin/env bash
    
   BASE_IMAGE=${BASE_IMAGE:-centos:7.8.2003}
   PY_PREFIX=${PY_PREFIX:-/opt/workfusion/python}
   PY_VER=${PY_VER:-3.7.9}
   OPT=${OPT:-noopt}
   WORK_DIR="`pwd`/_build-${PY_VER}"
   PRJ_DIR="python-build"
   PRJ_PATH="${WORK_DIR}/${PRJ_DIR}"
   DOCKERFILE_PATH="${WORK_DIR}/Dockerfile"
   BUILD_FLOW_IMAGE="build-flow:1.0"
   BUILD_FLOW_CONTAINER="build-${PY_VER}"
   _UID=`id -u`
   _GUID=`id -g`
    
    
   #signature()
   function error_handler() {
       local RET_CODE=$?
       echo "** CRITICAL ** Error was raised: ${RET_CODE}"
       exit ${RET_CODE}
   }
    
   # signature ()
   function generate_dockerfile() {
       cat > ${DOCKERFILE_PATH} << EOF
   FROM ${BASE_IMAGE}
   VOLUME /work-dir
    
   WORKDIR /work-dir
    
   # install system libs
   RUN yum update -y && yum -y install yum-utils
   RUN yum -y groupinstall development
   RUN yum install -y \\
     git \\
     wget \\
     gcc \\
     zlib-devel \\
     bzip2 \\
     bzip2-devel \\
     readline-devel \\
     sqlite \\
     sqlite-devel \\
     openssl-devel \\
     tk-devel \\
     libffi-devel \\
     xz-devel \\
     docker \\
     tar
    
   ENTRYPOINT ["/usr/bin/bash"]
   CMD []
   EOF
   }
    
    
   #signature()
   function build_image() {
       echo "** INFO ** Making dirs: ${PRJ_PATH}"
       mkdir -p ${PRJ_PATH} || error_handler
    
       echo "** INFO ** Generating dockerfile: ${DOCKERFILE_PATH}"
       generate_dockerfile \
       || error_handler
    
       echo "** INFO ** Build ${BUILD_FLOW_IMAGE} image"
       docker build -f ${DOCKERFILE_PATH} -t ${BUILD_FLOW_IMAGE} . \
       || error_handler
   }
    
   #signature
   function clone_project() {
       docker run -it --rm \
           -v ${WORK_DIR}:/work-dir:rw \
           ${BUILD_FLOW_IMAGE} \
           -c "wget https://www.python.org/ftp/python/${PY_VER}/Python-${PY_VER}.tgz -O /work-dir/${PRJ_DIR}/Python-${PY_VER}.tgz \
              && cd /work-dir/${PRJ_DIR} \
              && tar -zxvf Python-${PY_VER}.tgz"
   }
    
   #signature
   function build_cpython() {
       echo "** INFO ** Build python: ${BUILD_FLOW_CONTAINER}"
    
       docker rm ${BUILD_FLOW_CONTAINER}
       docker run -it \
           --name ${BUILD_FLOW_CONTAINER} \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -v ${WORK_DIR}:/work-dir:rw  \
           ${BUILD_FLOW_IMAGE} \
           -c "cd /work-dir/${PRJ_DIR}/Python-${PY_VER} \
              && ./configure --prefix=${PY_PREFIX}/python${PY_VER} --enable-optimizations \
              && make clean \
              && make \
              && make install \
              && mkdir -p /work-dir/output/ \
              && cp -R ${PY_PREFIX}/python${PY_VER} /work-dir/output/python${PY_VER}" \
           || error_handler
   }
    
   #signature
   function repack_and_chown() {
       echo "** INFO ** Repack and chown to (${_UID}:${_GUID}): ${BUILD_FLOW_CONTAINER}"
    
       docker rm ${BUILD_FLOW_CONTAINER}
       docker run -it \
           --name ${BUILD_FLOW_CONTAINER} \
           -v /var/run/docker.sock:/var/run/docker.sock \
           -v ${WORK_DIR}:/work-dir:rw  \
           ${BUILD_FLOW_IMAGE} \
           -c "cd /work-dir/output \
               && find -name __pycache__|xargs rm -rf \
               && chown -R ${_UID}:${_GUID} python${PY_VER} \
               && tar -czf python${PY_VER}.tgz python${PY_VER} \
               && chown -R ${_UID}:${_GUID} python${PY_VER}.tgz" \
           || error_handler
   }
    
   # signature()
   function build_flow() {
       build_image
       clone_project
       build_cpython
       repack_and_chown
       echo "** INFO ** Build completed. Check archive ${WORK_DIR}/output/python${PY_VER}.tgz"
   }
    
   #signature()
   function build_image_bash() {
       build_image
    
       echo "** INFO ** Launch container with bash"
       docker run -it --rm \
           -v ${WORK_DIR}:/work-dir:rw \
           ${BUILD_FLOW_IMAGE}
   }
    
   function usage() {
           echo """\
   ***************************************
   *********** Build CTL v.1.0 ***********
   ***************************************
   USAGE: $0 <command>
            command ::= build|bash|repack
   ENVIRONMENT:
           BASE_IMAGE=<base-image>
               base-image ::= docker-image (default: ${BASE_IMAGE})
    
           PY_VER=<cpython-ver>
               cpython-ver ::= 3.7.10|3.8.8|3.9.2 (default: ${PY_VER})
   EXAMPLES:
           1) ./python_builder.sh build
           2) BASE_IMAGE=centos:7.8.2003 PY_VER=3.7.10 ./python_builder.sh build
           3) ./python_builder.sh bash
    
   REMARK:
           chmod a+x python_builder.sh
   """
       exit 1
   }
    
    
   case $1 in
       build)
           time build_flow
           ;;
       bash)
           build_image_bash
           ;;
       repack)
           repack_and_chown
           ;;
       *)
           usage
       ;;
   esac

2. To build Python, run the following command:

   ./python_build.sh build

Fetch dependencies

There are two major sets of Python dependencies:

• Python packages
• Binary libraries used by the Python packages (bindings)

Python packages

To fetch Python packages, follow the instruction:

1. Provide an archive with prebuilt Python (python-<PY_VER>.tgz).

2. Go to the <WORK_DIR> working directory or create one if it doesn't exist yet.

3. Add the python-<PY_VER>.tgz Python archive to <WORK_DIR>.

4. Create the fetching script as shown below:

   Expand for deps_builder.sh code

   #!/usr/bin/env bash
    
   BASE_IMAGE=${BASE_IMAGE:-centos:7.8.2003}
   PY_VER=${PY_VER:-3.7.9}
   REQS=${REQS:-requirements.txt}
   WORK_DIR="`pwd`/_build-${PY_VER}"
   PRJ_DIR="wheels-build"
   PRJ_PATH="${WORK_DIR}/${PRJ_DIR}"
   DOCKERFILE_PATH="${WORK_DIR}/wheels.Dockerfile"
   BUILD_FLOW_IMAGE="build-flow:1.0"
   BUILD_FLOW_CONTAINER="build-${PY_VER}"
   _UID=`id -u`
   _GUID=`id -g`
    
    
   #signature()
   function error_handler() {
       local RET_CODE=$?
       echo "** CRITICAL ** Error was raised: ${RET_CODE}"
       exit ${RET_CODE}
   }
    
   # signature ()
   function generate_dockerfile() {
       cat > ${DOCKERFILE_PATH} << EOF
   FROM ${BASE_IMAGE}
   VOLUME /work-dir
    
   WORKDIR /work-dir
    
   # install system libs
   RUN yum update -y && yum -y install yum-utils
   RUN yum -y groupinstall development
   RUN yum install -y \\
     git \\
     wget \\
     gcc \\
     zlib-devel \\
     bzip2 \\
     bzip2-devel \\
     readline-devel \\
     sqlite \\
     sqlite-devel \\
     openssl-devel \\
     tk-devel \\
     libffi-devel \\
     xz-devel \\
     docker \\
     tar
    
   ENTRYPOINT ["/usr/bin/bash"]
   CMD []
   EOF
   }
    
    
   #signature()
   function build_image() {
       echo "** INFO ** Making dirs: ${PRJ_PATH}"
       mkdir -p ${PRJ_PATH} || error_handler
    
       echo "** INFO ** Generating dockerfile: ${DOCKERFILE_PATH}"
       generate_dockerfile \
       || error_handler
    
       echo "** INFO ** Build ${BUILD_FLOW_IMAGE} image"
       docker build -f ${DOCKERFILE_PATH} -t ${BUILD_FLOW_IMAGE} . \
       || error_handler
   }
    
   #signature
   function copy_python() {
       echo "** INFO ** Copy python: python-${PY_VER}.tgz -> ${PRJ_PATH}"
       cp "python${PY_VER}.tgz" "${PRJ_PATH}/python.tgz" \
       || error_handler
   }
    
   #signature
   function copy_reqs() {
       echo "** INFO ** Copy requirements: ${REQS} -> ${PRJ_PATH}"
       cp "$REQS" "${PRJ_PATH}/requirements.txt" \
       || error_handler
   }
    
   #signature
   function build_wheels() {
       echo "** INFO ** Build wheels: ${BUILD_FLOW_CONTAINER}"
    
       docker rm ${BUILD_FLOW_CONTAINER}
       docker run -it \
           --name ${BUILD_FLOW_CONTAINER} \
           -v ${WORK_DIR}:/work-dir:rw  \
           ${BUILD_FLOW_IMAGE} \
           -c "cd /work-dir/${PRJ_DIR} \
              && tar -zxvf python.tgz \
              && mkdir -p /work-dir/output/wheels \
              && ./python${PY_VER}/bin/python3 -m pip install wheel \
              && ./python${PY_VER}/bin/python3 -m pip wheel -r requirements.txt --wheel-dir /work-dir/output/wheels" \
           || error_handler
   }
    
   #signature
   function repack_and_chown() {
       echo "** INFO ** Repack and chown to (${_UID}:${_GUID}): ${BUILD_FLOW_CONTAINER}"
    
       docker rm ${BUILD_FLOW_CONTAINER}
       docker run -it \
           --name ${BUILD_FLOW_CONTAINER} \
           -v ${WORK_DIR}:/work-dir:rw  \
           ${BUILD_FLOW_IMAGE} \
           -c "cd /work-dir/output \
               && chown -R ${_UID}:${_GUID} wheels \
               && tar -cf wheels${PY_VER}.tar wheels \
               && chown -R ${_UID}:${_GUID} wheels${PY_VER}.tar" \
           || error_handler
   }
    
   # signature()
   function build_flow() {
       build_image
       copy_python
       copy_reqs
       build_wheels
       repack_and_chown
       echo "** INFO ** Build completed. Check archive ${WORK_DIR}/output/wheels${PY_VER}.tar"
   }
    
   #signature()
   function build_image_bash() {
       build_image
    
       echo "** INFO ** Launch container with bash"
       docker run -it --rm \
           -v ${WORK_DIR}:/work-dir:rw \
           ${BUILD_FLOW_IMAGE}
   }
    
   function usage() {
           echo """\
   ***************************************
   *********** Build CTL v.1.0 ***********
   ***************************************
   USAGE: $0 <command>
            command ::= build|bash|repack
   ENVIRONMENT:
           BASE_IMAGE=<base-image>
               base-image ::= docker-image (default: ${BASE_IMAGE})
    
           PY_VER=<cpython-ver>
               cpython-ver ::= 3.7.10|3.8.8|3.9.2 (default: ${PY_VER})
    
           REQS=<requirements>
               requirements ::= requirements.txt (default: ${REQS})
   EXAMPLES:
           1) ./deps_builder.sh build
           2) BASE_IMAGE=centos:7.8.2003 REQS=./requirements.txt ./deps_builder.sh build
           3) ./deps_builder.sh bash
    
   REMARK:
           chmod a+x python_builder.sh
   """
       exit 1
   }
    
    
   case $1 in
       build)
           time build_flow
           ;;
       bash)
           build_image_bash
           ;;
       repack)
           repack_and_chown
           ;;
       *)
           usage
       ;;
   esac

5. Build the dependencies by running the following command:

   ./deps_builder.sh build

Binary libraries

Add Python binary libraries to the directory located at LD_LIBRARY_PATH.

Prepare distribution bundle

Build a zip package with Python and wheel archives and deploy it to Agents.