diff --git a/workspaces/data-workspace/Dockerfile b/workspaces/data-workspace/Dockerfile index fc7cd5b..7b1f8ed 100755 --- a/workspaces/data-workspace/Dockerfile +++ b/workspaces/data-workspace/Dockerfile @@ -43,13 +43,24 @@ LABEL maintainer="Bluxmit " USER root +#COPY ./mkdocs/mkdocs.yml /home/docs/mkdocs.yml +COPY ./mkdocs/home.md /home/docs/docs/pages/home +COPY ./mkdocs/helpers.py /home/docs/macros +COPY ./mkdocs/Jupyter.png /home/docs/docs/pages/home/home/ +COPY ./mkdocs/Jupyterlab.png /home/docs/docs/pages/home/home/ +COPY ./mkdocs/Nbviewer.png /home/docs/docs/pages/home/home/ +COPY ./mkdocs/Luigi.png /home/docs/docs/pages/home/home/ + + ################################################################# LUIGI ENV LUIGI_VERSION="3.0.3" ARG LUIGI_CONFIG_DIR="/opt/luigi/" -COPY luigi /opt/ -RUN mkdir -p /etc/service/luigid/ +COPY ./luigi /opt/luigi/ +RUN mkdir -p /etc/service/luigid/ \ + && mkdir -p /var/log/luigi \ + && mkdir -p /var/log/jupyter ################################################################# JUPYTER @@ -199,6 +210,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \ ca-certificates \ libcurl4 \ git \ + libmemcached-dev \ && apt-get clean && rm -rf /var/lib/apt/lists/* \ && mkdir -p /home/nbviewer @@ -217,8 +229,10 @@ RUN chown -R abc /var/log/supervisord \ && mkdir -p ${LUIGI_CONFIG_DIR} \ && chown -R abc ${LUIGI_CONFIG_DIR} /etc/service/luigid/ \ && chown -R abc /home/abc/installed-python-packages \ - && chown -R abc /home/nbviewer - # && pip install -r /home/abc/installed-python-packages/data-workspace.txt + && chown -R abc /home/nbviewer \ + && chown -R abc /var/log/luigi \ + && chown -R abc /var/log/jupyter \ + && pip install -r /home/abc/installed-python-packages/data-workspace.txt USER abc diff --git a/workspaces/data-workspace/Dockerfile2 b/workspaces/data-workspace/Dockerfile2 new file mode 100755 index 0000000..5132fe9 --- /dev/null +++ b/workspaces/data-workspace/Dockerfile2 @@ -0,0 +1,197 @@ +ARG docker_registry=docker.io/alnoda +ARG parent_image=python-workspace +ARG parent_image_tag=3.9-0.2 + +FROM ${docker_registry}/${parent_image}:${parent_image_tag} +LABEL maintainer="Bluxmit " + +USER root + +#COPY ./mkdocs/mkdocs.yml /home/docs/mkdocs.yml +COPY ./mkdocs/home.md /home/docs/docs/pages/home +COPY ./mkdocs/helpers.py /home/docs/macros +COPY ./mkdocs/Jupyter.png /home/docs/docs/pages/home/home/ +COPY ./mkdocs/Jupyterlab.png /home/docs/docs/pages/home/home/ +COPY ./mkdocs/Nbviewer.png /home/docs/docs/pages/home/home/ + +################################################################# LUIGI + +ENV LUIGI_VERSION="3.0.3" +ARG LUIGI_CONFIG_DIR="/opt/luigi/" + +COPY ./luigi /opt/luigi/ +RUN mkdir -p /etc/service/luigid/ \ + && mkdir -p /var/log/jupyter + + +################################################################# JUPYTER + +ARG NB_USER="abc" +ARG NB_UID="1000" +ARG NB_GID="100" + +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for notebook server that starts but lacks all +# features (e.g., download as all possible file formats) +# Install tini: init for containers +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + wget \ + ca-certificates \ + sudo \ + locales \ + fonts-liberation \ + run-one && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH="${CONDA_DIR}/bin:${PATH}" \ + HOME="/home/${NB_USER}" + +# Copy a script that we will use to correct permissions after running certain commands +COPY ./jupyter/fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default NB_USER +# hadolint ignore=SC2016 +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ + # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 + echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + +# Create NB_USER with name abc user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + mkdir -p "${CONDA_DIR}" && \ + usermod -a -G ${NB_GID} ${NB_USER} && \ + chown -R abc /opt/conda && \ + chmod g+w /etc/passwd && \ + fix-permissions "${HOME}" && \ + fix-permissions "${CONDA_DIR}" + +USER ${NB_USER} +ARG PYTHON_VERSION=default + +# Setup work directory for backward-compatibility +RUN mkdir "/home/${NB_USER}/work" && \ + fix-permissions "/home/${NB_USER}" + +# Install conda as abc and check the sha256 sum provided on the download site +WORKDIR /tmp + +# ---- Miniforge installer ---- +# Check https://github.com/conda-forge/miniforge/releases +# Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge) +# We're using Mambaforge installer, possible options: +# - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy +# - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy +# Installation: conda, mamba, pip +RUN set -x && \ + # Miniforge installer + miniforge_arch=$(uname -m) && \ + miniforge_installer="Mambaforge-Linux-${miniforge_arch}.sh" && \ + wget --quiet "https://github.com/conda-forge/miniforge/releases/latest/download/${miniforge_installer}" && \ + /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ + rm "${miniforge_installer}" && \ + # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + conda config --system --set auto_update_conda false && \ + conda config --system --set show_channel_urls true && \ + if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \ + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + # Using conda to update all packages: https://github.com/mamba-org/mamba/issues/1092 + conda install --quiet --yes 'tini=0.18.0' && \ + conda update --all --quiet --yes && \ + conda clean --all -f -y && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Install Jupyter Notebook, Lab, and Hub +# Generate a notebook server config +# Cleanup temporary files +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +RUN mamba install --quiet --yes \ + 'notebook' \ + 'jupyterhub' \ + 'jupyterlab' && \ + mamba clean --all -f -y && \ + npm cache clean --force && \ + jupyter notebook --generate-config && \ + jupyter lab clean && \ + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Copy local files as late as possible to avoid cache busting +COPY ./jupyter/start.sh ./jupyter/start-notebook.sh ./jupyter/start-singleuser.sh /usr/local/bin/ +# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab +COPY ./jupyter/jupyter_notebook_config.py /etc/jupyter/ + +# Fix permissions on /etc/jupyter as root +USER root + +# Prepare upgrade to JupyterLab V3.0 #1205 +RUN sed -re "s/c.NotebookApp/c.ServerApp/g" \ + /etc/jupyter/jupyter_notebook_config.py > /etc/jupyter/jupyter_server_config.py && \ + fix-permissions /etc/jupyter/ + +RUN chmod 0777 /usr/local/bin/start-notebook.sh \ + && mkdir -p /var/log/supervisord + + +################################################################# NBVIEWER + +# To change the number of threads use env var NBVIEWER_THREADS +# docker run -d -e NBVIEWER_THREADS=4 -p 80:8080 nbviewer +ENV NBVIEWER_THREADS 2 # default - 2 threads +ENV LANG=C.UTF-8 + +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && apt-get install -yq --no-install-recommends \ + ca-certificates \ + libcurl4 \ + git \ + libmemcached-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /home/nbviewer + + +################################################################# PACKAGES & MODULES + +#RUN apt-get install -y default-libmysqlclient-dev build-essential + +COPY data-workspace.txt /home/abc/installed-python-packages/ +COPY data-workspace.conf /etc/supervisord/ + +################## USER + +RUN chown -R abc /var/log/supervisord \ + && chown -R abc /opt/conda /etc/jupyter \ + && mkdir -p ${LUIGI_CONFIG_DIR} \ + && chown -R abc ${LUIGI_CONFIG_DIR} /etc/service/luigid/ \ + && chown -R abc /home/abc/installed-python-packages \ + && chown -R abc /home/nbviewer \ + && chown -R abc /var/log/jupyter \ + && pip install -r /home/abc/installed-python-packages/data-workspace.txt + +USER abc + + diff --git a/workspaces/data-workspace/data-workspace.conf b/workspaces/data-workspace/data-workspace.conf index 52f81c1..3310f78 100755 --- a/workspaces/data-workspace/data-workspace.conf +++ b/workspaces/data-workspace/data-workspace.conf @@ -1,33 +1,33 @@ [program:luigi] directory=/opt/luigi/ -command=/bin/sh -c " sh luigid.sh " -stderr_logfile = /var/log/luigi-stderr.log -stdout_logfile = /var/log/luigi-stdout.log +command=/bin/sh -c " sh /opt/luigi/luigid.sh " +stderr_logfile = /var/log/luigi/luigi-stderr.log +stdout_logfile = /var/log/luigi/luigi-stdout.log logfile_maxbytes = 1024 [program:jupyter] directory=/usr/local/bin/ -command=jupyter notebook --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/notebooks --no-browser --port=8031 -stderr_logfile = /var/log/jupyter-stderr.log -stdout_logfile = /var/log/jupyter-stdout.log +command=jupyter notebook --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/ --no-browser --port=8031 +stderr_logfile = /var/log/jupyter/jupyter-stderr.log +stdout_logfile = /var/log/jupyter/jupyter-stdout.log logfile_maxbytes = 1024 [program:jupytelab] directory=/usr/local/bin/ -command=jupyter lab --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/notebooks --no-browser --port=8032 -stderr_logfile = /var/log/jupyterlab-stderr.log -stdout_logfile = /var/log/jupyterlab-stdout.log +command=jupyter lab --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/ --no-browser --port=8032 +stderr_logfile = /var/log/jupyter/jupyterlab-stderr.log +stdout_logfile = /var/log/jupyter/jupyterlab-stdout.log logfile_maxbytes = 1024 [program:nbviewer] directory=/usr/local/bin/ command=python -m nbviewer --port=8033 --localfiles=/home/nbviewer -stderr_logfile = /var/log/nbviewer-stderr.log -stdout_logfile = /var/log/nbviewer-stdout.log +stderr_logfile = /var/log/jupyter/nbviewer-stderr.log +stdout_logfile = /var/log/jupyter/nbviewer-stdout.log logfile_maxbytes = 1024 diff --git a/workspaces/data-workspace/data-workspace.txt b/workspaces/data-workspace/data-workspace.txt index badfc3a..2d98d36 100755 --- a/workspaces/data-workspace/data-workspace.txt +++ b/workspaces/data-workspace/data-workspace.txt @@ -17,7 +17,7 @@ fsspec==2021.9.0 s3fs==2021.9.0 ## Notebooks -markdown==3.3.4 +#markdown==3.3.4 nbconvert==6.2.0 nbformat==5.1.3 statsd==3.3.0 @@ -25,6 +25,10 @@ tornado==6.1 papermill==2.3.3 papermill-report==2.0.3 +## Nbviewer +pylibmc==1.6.1 +#nbviewer==1.0.1 + ## More Jupyter # https://pypi.org/project/jupytext/ jupytext==1.13.0 @@ -41,12 +45,11 @@ scrapbook==0.5.0 # https://pypi.org/project/nbtoolbelt/ nbtoolbelt==2020.7.dev2 # https://pypi.org/project/mkdocs-jupyter/ -mkdocs-jupyter==0.17.5 +#mkdocs-jupyter==0.17.5 # https://pypi.org/project/notebooker/ -notebooker==0.2.0 +#notebooker==0.2.0 # https://pypi.org/project/nbless/ nbless==0.2.38 ## Canncelled # elasticsearch==7.13.1 -# pylibmc==1.6.1 # failed \ No newline at end of file diff --git a/workspaces/data-workspace/mkdocs/Jupyter.png b/workspaces/data-workspace/mkdocs/Jupyter.png new file mode 100644 index 0000000..76d3b8f Binary files /dev/null and b/workspaces/data-workspace/mkdocs/Jupyter.png differ diff --git a/workspaces/data-workspace/mkdocs/Jupyterlab.png b/workspaces/data-workspace/mkdocs/Jupyterlab.png new file mode 100644 index 0000000..9426fd2 Binary files /dev/null and b/workspaces/data-workspace/mkdocs/Jupyterlab.png differ diff --git a/workspaces/data-workspace/mkdocs/Luigi.png b/workspaces/data-workspace/mkdocs/Luigi.png new file mode 100644 index 0000000..5f39674 Binary files /dev/null and b/workspaces/data-workspace/mkdocs/Luigi.png differ diff --git a/workspaces/data-workspace/mkdocs/Nbviewer.png b/workspaces/data-workspace/mkdocs/Nbviewer.png new file mode 100644 index 0000000..3723406 Binary files /dev/null and b/workspaces/data-workspace/mkdocs/Nbviewer.png differ diff --git a/workspaces/data-workspace/mkdocs/helpers.py b/workspaces/data-workspace/mkdocs/helpers.py new file mode 100644 index 0000000..9b0901a --- /dev/null +++ b/workspaces/data-workspace/mkdocs/helpers.py @@ -0,0 +1,61 @@ +""" +Basic example of a Mkdocs-macros module. +Include this {{ macros_info() }} in any page to get complete macro info +""" +import os + +port_increments = { + "DOCS_URL": 0, + "FILEBROWSER_URL": 1, + "STATICFS_URL": 2, + "CRONICLE_URL": 3, + "UNGIT_URL": 4, + "IDE_URL": 5, + "TERMINAL_URL": 6, + "MC_URL": 7, + "HTOP_URL": 8, + "LUIGI": 10, + "JUPYTER": 11, + "JUPYTERLAB": 12, + "NBVIEWER": 13 + } + +# this function name should not be changed +def define_env(env): + """ + This is the hook for defining variables, macros and filters + - variables: the dictionary that contains the environment variables + - macro: a decorator function, to declare a macro. + - filter: a function with one of more arguments, + used to perform a transformation + """ + @env.macro + def get_tool_url(env): + try: + return os.environ[name] + except: + # Get host + host = "localhost" + try: + host = os.environ["WRK_HOST"] + except: + pass + proto = "http" + try: + proto = os.environ["WRK_PROTO"] + except: + pass + # Entry port - port relative to which other ports will be calculated + entry_port = 8020 + try: + entry_port = int(os.environ["ENTRY_PORT"]) + except: + pass + # Assign port + try: + port = port_increments[env] + entry_port + except: + port = 80 + return f"{proto}://{host}:{port}" + + \ No newline at end of file diff --git a/workspaces/data-workspace/mkdocs/home.md b/workspaces/data-workspace/mkdocs/home.md new file mode 100644 index 0000000..14f4d0d --- /dev/null +++ b/workspaces/data-workspace/mkdocs/home.md @@ -0,0 +1,151 @@ + + + +{% + set tools = [ + { + "env": "IDE_URL", + "name": "IDE", + "image": "IDE.jpg", + "description": "Browser-based version of Visual Studio Code. Develop in any language, install hundreeds of extensions" + }, + { + "env": "TERMINAL_URL", + "name": "Terminal", + "image": "Terminal.png", + "description": "Full-fledged browser-based terminal with Z-shell" + }, + { + "env": "FILEBROWSER_URL", + "name": "File Browser", + "image": "Filebrowser.png", + "description": "Browse, upload and download files and folders to and from the Workspace" + }, + { + "env": "CRONICLE_URL", + "name": "Cronicle", + "image": "Cronicle.jpg", + "description": "Schedule jobs, manage schedules, observe and monitor executions (user/pass - admin/admin)" + }, + { + "env": "UNGIT_URL", + "name": "Ungit", + "image": "Ungit.jpg", + "description": "Manage Git repositories and work flow using beautiful UI" + }, + { + "env": "STATICFS_URL", + "name": "Static File Server", + "image": "Static-server.png", + "description": "Serve any static websites like a breeze" + }, + { + "env": "MC_URL", + "name": "M.Commander", + "image": "MC.jpg", + "description": "Feature rich visual file manager with internal text viewer and editor" + }, + { + "env": "HTOP_URL", + "name": "Process monitor", + "image": "Htop.jpg", + "description": "Monitor running process and resource utilization" + }, + { + "env": "LUIGI", + "name": "Luigi", + "image": "Luigi.png", + "description": "Monitor pipelines and batch jobs" + }, + { + "env": "JUPYTER", + "name": "Jupyter notebook", + "image": "Jupyter.png", + "description": "Jupyter notebook" + }, + { + "env": "JUPYTERLAB", + "name": "Jupyterlab", + "image": "Jupyterlab.png", + "description": "Jupyter lab" + }, + { + "env": "NBVIEWER", + "name": "Nbviewer", + "image": "Nbviewer.png", + "description": "Nbviewer" + } + ] +%} + + +
+ {% for tool in tools %} + {% set tool_url = get_tool_url(tool.env) %} +
+ + + + +
{{ tool.name }}
+ +
{{ tool.description }}
+
+ {% endfor %} +
+ + + + +