10.2.2

Edit

Prepare Python

To prepare a distribution bundle for setting up a custom environment in a cluster on an Agent, build Python and fetch dependencies as described below.

Build Python

To ensure the required non-root, or standalone, installation of Python, meet the prerequisites below and build Python from the source code.

Meet prerequisites

  • Install and configure Docker.
  • Install bash.
  • Make sure your Internet connection is ok.

Build Python from source code

Follow the instruction:

  1. Create the build script as shown below:

    Expand for python_builder.sh code

    #!/usr/bin/env bash
 
BASE_IMAGE=${BASE_IMAGE:-centos:7.8.2003}
PY_PREFIX=${PY_PREFIX:-/opt/workfusion/python}
PY_VER=${PY_VER:-3.7.9}
OPT=${OPT:-noopt}
WORK_DIR="`pwd`/_build-${PY_VER}"
PRJ_DIR="python-build"
PRJ_PATH="${WORK_DIR}/${PRJ_DIR}"
DOCKERFILE_PATH="${WORK_DIR}/Dockerfile"
BUILD_FLOW_IMAGE="build-flow:1.0"
BUILD_FLOW_CONTAINER="build-${PY_VER}"
_UID=`id -u`
_GUID=`id -g`
 
 
#signature()
function error_handler() {
    local RET_CODE=$?
    echo "** CRITICAL ** Error was raised: ${RET_CODE}"
    exit ${RET_CODE}
}
 
# signature ()
function generate_dockerfile() {
    cat > ${DOCKERFILE_PATH} << EOF
FROM ${BASE_IMAGE}
VOLUME /work-dir
 
WORKDIR /work-dir
 
# install system libs
RUN yum update -y && yum -y install yum-utils
RUN yum -y groupinstall development
RUN yum install -y \\
  git \\
  wget \\
  gcc \\
  zlib-devel \\
  bzip2 \\
  bzip2-devel \\
  readline-devel \\
  sqlite \\
  sqlite-devel \\
  openssl-devel \\
  tk-devel \\
  libffi-devel \\
  xz-devel \\
  docker \\
  tar
 
ENTRYPOINT ["/usr/bin/bash"]
CMD []
EOF
}
 
 
#signature()
function build_image() {
    echo "** INFO ** Making dirs: ${PRJ_PATH}"
    mkdir -p ${PRJ_PATH} || error_handler
 
    echo "** INFO ** Generating dockerfile: ${DOCKERFILE_PATH}"
    generate_dockerfile \
    || error_handler
 
    echo "** INFO ** Build ${BUILD_FLOW_IMAGE} image"
    docker build -f ${DOCKERFILE_PATH} -t ${BUILD_FLOW_IMAGE} . \
    || error_handler
}
 
#signature
function clone_project() {
    docker run -it --rm \
        -v ${WORK_DIR}:/work-dir:rw \
        ${BUILD_FLOW_IMAGE} \
        -c "wget https://www.python.org/ftp/python/${PY_VER}/Python-${PY_VER}.tgz -O /work-dir/${PRJ_DIR}/Python-${PY_VER}.tgz \
           && cd /work-dir/${PRJ_DIR} \
           && tar -zxvf Python-${PY_VER}.tgz"
}
 
#signature
function build_cpython() {
    echo "** INFO ** Build python: ${BUILD_FLOW_CONTAINER}"
 
    docker rm ${BUILD_FLOW_CONTAINER}
    docker run -it \
        --name ${BUILD_FLOW_CONTAINER} \
        -v /var/run/docker.sock:/var/run/docker.sock \
        -v ${WORK_DIR}:/work-dir:rw  \
        ${BUILD_FLOW_IMAGE} \
        -c "cd /work-dir/${PRJ_DIR}/Python-${PY_VER} \
           && ./configure --prefix=${PY_PREFIX}/python${PY_VER} --enable-optimizations \
           && make clean \
           && make \
           && make install \
           && mkdir -p /work-dir/output/ \
           && cp -R ${PY_PREFIX}/python${PY_VER} /work-dir/output/python${PY_VER}" \
        || error_handler
}
 
#signature
function repack_and_chown() {
    echo "** INFO ** Repack and chown to (${_UID}:${_GUID}): ${BUILD_FLOW_CONTAINER}"
 
    docker rm ${BUILD_FLOW_CONTAINER}
    docker run -it \
        --name ${BUILD_FLOW_CONTAINER} \
        -v /var/run/docker.sock:/var/run/docker.sock \
        -v ${WORK_DIR}:/work-dir:rw  \
        ${BUILD_FLOW_IMAGE} \
        -c "cd /work-dir/output \
            && find -name __pycache__|xargs rm -rf \
            && chown -R ${_UID}:${_GUID} python${PY_VER} \
            && tar -czf python${PY_VER}.tgz python${PY_VER} \
            && chown -R ${_UID}:${_GUID} python${PY_VER}.tgz" \
        || error_handler
}
 
# signature()
function build_flow() {
    build_image
    clone_project
    build_cpython
    repack_and_chown
    echo "** INFO ** Build completed. Check archive ${WORK_DIR}/output/python${PY_VER}.tgz"
}
 
#signature()
function build_image_bash() {
    build_image
 
    echo "** INFO ** Launch container with bash"
    docker run -it --rm \
        -v ${WORK_DIR}:/work-dir:rw \
        ${BUILD_FLOW_IMAGE}
}
 
function usage() {
        echo """\
***************************************
*********** Build CTL v.1.0 ***********
***************************************
USAGE: $0 <command>
         command ::= build|bash|repack
ENVIRONMENT:
        BASE_IMAGE=<base-image>
            base-image ::= docker-image (default: ${BASE_IMAGE})
 
        PY_VER=<cpython-ver>
            cpython-ver ::= 3.7.10|3.8.8|3.9.2 (default: ${PY_VER})
EXAMPLES:
        1) ./python_builder.sh build
        2) BASE_IMAGE=centos:7.8.2003 PY_VER=3.7.10 ./python_builder.sh build
        3) ./python_builder.sh bash
 
REMARK:
        chmod a+x python_builder.sh
"""
    exit 1
}
 
 
case $1 in
    build)
        time build_flow
        ;;
    bash)
        build_image_bash
        ;;
    repack)
        repack_and_chown
        ;;
    *)
        usage
    ;;
esac

  2. To build Python, run the following command:

    ./deps_builder.sh build

Fetch dependencies

There are two major sets of Python dependencies:

  • Python packages
  • binary libraries used by the Python packages (bindings)

Python packages

To fetch Python packages, follow the instruction:

  1. Provide an archive with prebuilt Python (python-<PY_VER>.tgz).

  2. Go to the <WORK_DIR> working directory or create one if it doesn't exist yet.

  3. Add the python-<PY_VER>.tgz Python archive to <WORK_DIR>.

  4. Create the fetching script as shown below:

    Expand for deps_builder.sh code

    #!/usr/bin/env bash
 
BASE_IMAGE=${BASE_IMAGE:-centos:7.8.2003}
PY_VER=${PY_VER:-3.7.9}
REQS=${REQS:-requirements.txt}
WORK_DIR="`pwd`/_build-${PY_VER}"
PRJ_DIR="wheels-build"
PRJ_PATH="${WORK_DIR}/${PRJ_DIR}"
DOCKERFILE_PATH="${WORK_DIR}/wheels.Dockerfile"
BUILD_FLOW_IMAGE="build-flow:1.0"
BUILD_FLOW_CONTAINER="build-${PY_VER}"
_UID=`id -u`
_GUID=`id -g`
 
 
#signature()
function error_handler() {
    local RET_CODE=$?
    echo "** CRITICAL ** Error was raised: ${RET_CODE}"
    exit ${RET_CODE}
}
 
# signature ()
function generate_dockerfile() {
    cat > ${DOCKERFILE_PATH} << EOF
FROM ${BASE_IMAGE}
VOLUME /work-dir
 
WORKDIR /work-dir
 
# install system libs
RUN yum update -y && yum -y install yum-utils
RUN yum -y groupinstall development
RUN yum install -y \\
  git \\
  wget \\
  gcc \\
  zlib-devel \\
  bzip2 \\
  bzip2-devel \\
  readline-devel \\
  sqlite \\
  sqlite-devel \\
  openssl-devel \\
  tk-devel \\
  libffi-devel \\
  xz-devel \\
  docker \\
  tar
 
ENTRYPOINT ["/usr/bin/bash"]
CMD []
EOF
}
 
 
#signature()
function build_image() {
    echo "** INFO ** Making dirs: ${PRJ_PATH}"
    mkdir -p ${PRJ_PATH} || error_handler
 
    echo "** INFO ** Generating dockerfile: ${DOCKERFILE_PATH}"
    generate_dockerfile \
    || error_handler
 
    echo "** INFO ** Build ${BUILD_FLOW_IMAGE} image"
    docker build -f ${DOCKERFILE_PATH} -t ${BUILD_FLOW_IMAGE} . \
    || error_handler
}
 
#signature
function copy_python() {
    echo "** INFO ** Copy python: python-${PY_VER}.tgz -> ${PRJ_PATH}"
    cp "python${PY_VER}.tgz" "${PRJ_PATH}/python.tgz" \
    || error_handler
}
 
#signature
function copy_reqs() {
    echo "** INFO ** Copy requirements: ${REQS} -> ${PRJ_PATH}"
    cp "$REQS" "${PRJ_PATH}/requirements.txt" \
    || error_handler
}
 
#signature
function build_wheels() {
    echo "** INFO ** Build wheels: ${BUILD_FLOW_CONTAINER}"
 
    docker rm ${BUILD_FLOW_CONTAINER}
    docker run -it \
        --name ${BUILD_FLOW_CONTAINER} \
        -v ${WORK_DIR}:/work-dir:rw  \
        ${BUILD_FLOW_IMAGE} \
        -c "cd /work-dir/${PRJ_DIR} \
           && tar -zxvf python.tgz \
           && mkdir -p /work-dir/output/wheels \
           && ./python${PY_VER}/bin/python3 -m pip install wheel \
           && ./python${PY_VER}/bin/python3 -m pip wheel -r requirements.txt --wheel-dir /work-dir/output/wheels" \
        || error_handler
}
 
#signature
function repack_and_chown() {
    echo "** INFO ** Repack and chown to (${_UID}:${_GUID}): ${BUILD_FLOW_CONTAINER}"
 
    docker rm ${BUILD_FLOW_CONTAINER}
    docker run -it \
        --name ${BUILD_FLOW_CONTAINER} \
        -v ${WORK_DIR}:/work-dir:rw  \
        ${BUILD_FLOW_IMAGE} \
        -c "cd /work-dir/output \
            && chown -R ${_UID}:${_GUID} wheels \
            && tar -cf wheels${PY_VER}.tar wheels \
            && chown -R ${_UID}:${_GUID} wheels${PY_VER}.tar" \
        || error_handler
}
 
# signature()
function build_flow() {
    build_image
    copy_python
    copy_reqs
    build_wheels
    repack_and_chown
    echo "** INFO ** Build completed. Check archive ${WORK_DIR}/output/wheels${PY_VER}.tar"
}
 
#signature()
function build_image_bash() {
    build_image
 
    echo "** INFO ** Launch container with bash"
    docker run -it --rm \
        -v ${WORK_DIR}:/work-dir:rw \
        ${BUILD_FLOW_IMAGE}
}
 
function usage() {
        echo """\
***************************************
*********** Build CTL v.1.0 ***********
***************************************
USAGE: $0 <command>
         command ::= build|bash|repack
ENVIRONMENT:
        BASE_IMAGE=<base-image>
            base-image ::= docker-image (default: ${BASE_IMAGE})
 
        PY_VER=<cpython-ver>
            cpython-ver ::= 3.7.10|3.8.8|3.9.2 (default: ${PY_VER})
 
        REQS=<requirements>
            requirements ::= requirements.txt (default: ${REQS})
EXAMPLES:
        1) ./deps_builder.sh build
        2) BASE_IMAGE=centos:7.8.2003 REQS=./requirements.txt ./deps_builder.sh build
        3) ./deps_builder.sh bash
 
REMARK:
        chmod a+x python_builder.sh
"""
    exit 1
}
 
 
case $1 in
    build)
        time build_flow
        ;;
    bash)
        build_image_bash
        ;;
    repack)
        repack_and_chown
        ;;
    *)
        usage
    ;;
esac

  5. Build the dependencies by running the following command:

    ./deps_builder.sh build

Binary libraries

Add Python binary libraries to the directory located at LD_LIBRARY_PATH.

Prepare distribution bundle

Build a ZIP package with Python and wheel archives and deploy it to Agents.

← OverviewDeploy Python environment and configure IA Cloud to use it →