working on data-workspace

workspaces/data-workspace/Dockerfile

@@ -43,13 +43,24 @@ LABEL maintainer="Bluxmit <bluxmit@gmail.com>"
 
 USER root
 
+#COPY ./mkdocs/mkdocs.yml /home/docs/mkdocs.yml
+COPY ./mkdocs/home.md /home/docs/docs/pages/home
+COPY ./mkdocs/helpers.py /home/docs/macros 
+COPY ./mkdocs/Jupyter.png /home/docs/docs/pages/home/home/
+COPY ./mkdocs/Jupyterlab.png /home/docs/docs/pages/home/home/
+COPY ./mkdocs/Nbviewer.png /home/docs/docs/pages/home/home/
+COPY ./mkdocs/Luigi.png /home/docs/docs/pages/home/home/
+
+
 ################################################################# LUIGI
 
 ENV LUIGI_VERSION="3.0.3"
 ARG LUIGI_CONFIG_DIR="/opt/luigi/"
 
-COPY luigi /opt/
-RUN mkdir -p /etc/service/luigid/
+COPY ./luigi /opt/luigi/
+RUN mkdir -p /etc/service/luigid/ \
+    && mkdir -p /var/log/luigi \
+    && mkdir -p /var/log/jupyter
 
 
 ################################################################# JUPYTER
@@ -199,6 +210,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
         ca-certificates \
         libcurl4 \
         git \
+        libmemcached-dev \
     && apt-get clean && rm -rf /var/lib/apt/lists/* \
     && mkdir -p /home/nbviewer
 
@@ -217,8 +229,10 @@ RUN chown -R abc /var/log/supervisord \
     && mkdir -p ${LUIGI_CONFIG_DIR} \
     && chown -R abc ${LUIGI_CONFIG_DIR} /etc/service/luigid/ \
     && chown -R abc /home/abc/installed-python-packages \
-    && chown -R abc /home/nbviewer 
-   # && pip install -r /home/abc/installed-python-packages/data-workspace.txt 
+    && chown -R abc /home/nbviewer \
+    && chown -R abc /var/log/luigi \
+    && chown -R abc /var/log/jupyter \
+    && pip install -r /home/abc/installed-python-packages/data-workspace.txt 
 
 USER abc
 

workspaces/data-workspace/Dockerfile2

@@ -0,0 +1,197 @@
+ARG docker_registry=docker.io/alnoda
+ARG parent_image=python-workspace
+ARG parent_image_tag=3.9-0.2 
+
+FROM ${docker_registry}/${parent_image}:${parent_image_tag}
+LABEL maintainer="Bluxmit <bluxmit@gmail.com>"
+
+USER root
+
+#COPY ./mkdocs/mkdocs.yml /home/docs/mkdocs.yml
+COPY ./mkdocs/home.md /home/docs/docs/pages/home
+COPY ./mkdocs/helpers.py /home/docs/macros 
+COPY ./mkdocs/Jupyter.png /home/docs/docs/pages/home/home/
+COPY ./mkdocs/Jupyterlab.png /home/docs/docs/pages/home/home/
+COPY ./mkdocs/Nbviewer.png /home/docs/docs/pages/home/home/
+
+################################################################# LUIGI
+
+ENV LUIGI_VERSION="3.0.3"
+ARG LUIGI_CONFIG_DIR="/opt/luigi/"
+
+COPY ./luigi /opt/luigi/
+RUN mkdir -p /etc/service/luigid/ \
+    && mkdir -p /var/log/jupyter
+
+
+################################################################# JUPYTER
+
+ARG NB_USER="abc"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+# Fix DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+# Install tini: init for containers
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update --yes && \
+    apt-get install --yes --no-install-recommends \
+    wget \
+    ca-certificates \
+    sudo \
+    locales \
+    fonts-liberation \
+    run-one && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen 
+
+# Configure environment
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    NB_USER="${NB_USER}" \
+    NB_UID=${NB_UID} \
+    NB_GID=${NB_GID} \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH="${CONDA_DIR}/bin:${PATH}" \
+    HOME="/home/${NB_USER}"
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY ./jupyter/fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+# Enable prompt color in the skeleton .bashrc before creating the default NB_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create NB_USER with name abc user with UID=1000 and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    mkdir -p "${CONDA_DIR}" && \
+    usermod -a -G ${NB_GID} ${NB_USER} && \
+    chown -R abc /opt/conda && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER ${NB_USER}
+ARG PYTHON_VERSION=default
+
+# Setup work directory for backward-compatibility
+RUN mkdir "/home/${NB_USER}/work" && \
+    fix-permissions "/home/${NB_USER}"
+
+# Install conda as abc and check the sha256 sum provided on the download site
+WORKDIR /tmp
+
+# ---- Miniforge installer ----
+# Check https://github.com/conda-forge/miniforge/releases
+# Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge)
+# We're using Mambaforge installer, possible options:
+# - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy
+# - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy
+# Installation: conda, mamba, pip
+RUN set -x && \
+    # Miniforge installer
+    miniforge_arch=$(uname -m) && \
+    miniforge_installer="Mambaforge-Linux-${miniforge_arch}.sh" && \
+    wget --quiet "https://github.com/conda-forge/miniforge/releases/latest/download/${miniforge_installer}" && \
+    /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \
+    rm "${miniforge_installer}" && \
+    # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html
+    conda config --system --set auto_update_conda false && \
+    conda config --system --set show_channel_urls true && \
+    if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \
+    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    # Using conda to update all packages: https://github.com/mamba-org/mamba/issues/1092
+    conda install --quiet --yes 'tini=0.18.0' && \
+    conda update --all --quiet --yes && \
+    conda clean --all -f -y && \
+    rm -rf "/home/${NB_USER}/.cache/yarn" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+# Install Jupyter Notebook, Lab, and Hub
+# Generate a notebook server config
+# Cleanup temporary files
+# Correct permissions
+# Do all this in a single RUN command to avoid duplicating all of the
+# files across image layers when the permissions change
+RUN mamba install --quiet --yes \
+    'notebook' \
+    'jupyterhub' \
+    'jupyterlab' && \
+    mamba clean --all -f -y && \
+    npm cache clean --force && \
+    jupyter notebook --generate-config && \
+    jupyter lab clean && \
+    rm -rf "/home/${NB_USER}/.cache/yarn" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+# Copy local files as late as possible to avoid cache busting
+COPY ./jupyter/start.sh ./jupyter/start-notebook.sh ./jupyter/start-singleuser.sh /usr/local/bin/
+# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab
+COPY ./jupyter/jupyter_notebook_config.py /etc/jupyter/
+
+# Fix permissions on /etc/jupyter as root
+USER root
+
+# Prepare upgrade to JupyterLab V3.0 #1205
+RUN sed -re "s/c.NotebookApp/c.ServerApp/g" \
+    /etc/jupyter/jupyter_notebook_config.py > /etc/jupyter/jupyter_server_config.py && \
+    fix-permissions /etc/jupyter/
+
+RUN chmod 0777 /usr/local/bin/start-notebook.sh \
+    && mkdir -p /var/log/supervisord 
+
+
+################################################################# NBVIEWER
+
+# To change the number of threads use env var NBVIEWER_THREADS
+# docker run -d -e NBVIEWER_THREADS=4 -p 80:8080 nbviewer
+ENV NBVIEWER_THREADS 2          # default - 2 threads
+ENV LANG=C.UTF-8
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && apt-get install -yq --no-install-recommends \
+        ca-certificates \
+        libcurl4 \
+        git \
+        libmemcached-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /home/nbviewer
+
+
+################################################################# PACKAGES & MODULES
+
+#RUN apt-get install -y default-libmysqlclient-dev build-essential
+
+COPY data-workspace.txt /home/abc/installed-python-packages/
+COPY data-workspace.conf /etc/supervisord/
+
+################## USER
+
+RUN chown -R abc /var/log/supervisord \
+    && chown -R abc /opt/conda /etc/jupyter \
+    && mkdir -p ${LUIGI_CONFIG_DIR} \
+    && chown -R abc ${LUIGI_CONFIG_DIR} /etc/service/luigid/ \
+    && chown -R abc /home/abc/installed-python-packages \
+    && chown -R abc /home/nbviewer \
+    && chown -R abc /var/log/jupyter \
+    && pip install -r /home/abc/installed-python-packages/data-workspace.txt 
+
+USER abc
+
+

workspaces/data-workspace/data-workspace.conf

@@ -1,33 +1,33 @@
 
 [program:luigi]
 directory=/opt/luigi/
-command=/bin/sh -c " sh luigid.sh "
-stderr_logfile = /var/log/luigi-stderr.log
-stdout_logfile = /var/log/luigi-stdout.log
+command=/bin/sh -c " sh /opt/luigi/luigid.sh "
+stderr_logfile = /var/log/luigi/luigi-stderr.log
+stdout_logfile = /var/log/luigi/luigi-stdout.log
 logfile_maxbytes = 1024
 
 
 [program:jupyter]
 directory=/usr/local/bin/
-command=jupyter notebook --allow-root --ip='*'  --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/notebooks --no-browser --port=8031
-stderr_logfile = /var/log/jupyter-stderr.log
-stdout_logfile = /var/log/jupyter-stdout.log
+command=jupyter notebook --allow-root --ip='*'  --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/ --no-browser --port=8031
+stderr_logfile = /var/log/jupyter/jupyter-stderr.log
+stdout_logfile = /var/log/jupyter/jupyter-stdout.log
 logfile_maxbytes = 1024
 
 
 [program:jupytelab]
 directory=/usr/local/bin/
-command=jupyter lab --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/notebooks --no-browser --port=8032
-stderr_logfile = /var/log/jupyterlab-stderr.log
-stdout_logfile = /var/log/jupyterlab-stdout.log
+command=jupyter lab --allow-root --ip='*' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/project/ --no-browser --port=8032
+stderr_logfile = /var/log/jupyter/jupyterlab-stderr.log
+stdout_logfile = /var/log/jupyter/jupyterlab-stdout.log
 logfile_maxbytes = 1024
 
 
 [program:nbviewer]
 directory=/usr/local/bin/
 command=python -m nbviewer --port=8033 --localfiles=/home/nbviewer
-stderr_logfile = /var/log/nbviewer-stderr.log
-stdout_logfile = /var/log/nbviewer-stdout.log
+stderr_logfile = /var/log/jupyter/nbviewer-stderr.log
+stdout_logfile = /var/log/jupyter/nbviewer-stdout.log
 logfile_maxbytes = 1024
 
 

workspaces/data-workspace/data-workspace.txt

@@ -17,7 +17,7 @@ fsspec==2021.9.0
 s3fs==2021.9.0 
 
 ## Notebooks
-markdown==3.3.4
+#markdown==3.3.4
 nbconvert==6.2.0
 nbformat==5.1.3
 statsd==3.3.0
@@ -25,6 +25,10 @@ tornado==6.1
 papermill==2.3.3
 papermill-report==2.0.3
 
+## Nbviewer
+pylibmc==1.6.1 
+#nbviewer==1.0.1
+
 ## More Jupyter
 # https://pypi.org/project/jupytext/
 jupytext==1.13.0 
@@ -41,12 +45,11 @@ scrapbook==0.5.0
 # https://pypi.org/project/nbtoolbelt/
 nbtoolbelt==2020.7.dev2
 # https://pypi.org/project/mkdocs-jupyter/
-mkdocs-jupyter==0.17.5
+#mkdocs-jupyter==0.17.5
 # https://pypi.org/project/notebooker/
-notebooker==0.2.0
+#notebooker==0.2.0
 # https://pypi.org/project/nbless/
 nbless==0.2.38
 
 ## Canncelled
 # elasticsearch==7.13.1
-# pylibmc==1.6.1            # failed

workspaces/data-workspace/mkdocs/helpers.py

@@ -0,0 +1,61 @@
+"""
+Basic example of a Mkdocs-macros module.
+Include this  {{ macros_info() }} in any page to get complete macro info
+"""
+import os
+
+port_increments = {
+    "DOCS_URL": 0,
+    "FILEBROWSER_URL": 1,
+    "STATICFS_URL": 2,
+    "CRONICLE_URL": 3,
+    "UNGIT_URL": 4,
+    "IDE_URL": 5,
+    "TERMINAL_URL": 6,
+    "MC_URL": 7,
+    "HTOP_URL": 8,
+    "LUIGI": 10,
+    "JUPYTER": 11,
+    "JUPYTERLAB": 12,
+    "NBVIEWER": 13
+    }
+
+# this function name should not be changed
+def define_env(env):
+    """
+    This is the hook for defining variables, macros and filters
+    - variables: the dictionary that contains the environment variables
+    - macro: a decorator function, to declare a macro.
+    - filter: a function with one of more arguments,
+        used to perform a transformation
+    """
+    @env.macro
+    def get_tool_url(env):
+        try:
+            return os.environ[name]
+        except:
+            # Get host
+            host = "localhost"
+            try:
+                host = os.environ["WRK_HOST"]
+            except:
+                pass
+            proto = "http"
+            try:
+                proto = os.environ["WRK_PROTO"]
+            except:
+                pass
+            # Entry port - port relative to which other ports will be calculated 
+            entry_port = 8020
+            try:
+                entry_port = int(os.environ["ENTRY_PORT"])
+            except:
+                pass
+            # Assign port
+            try:
+                port = port_increments[env] + entry_port
+            except:
+                port = 80
+            return f"{proto}://{host}:{port}"
+
+            

workspaces/data-workspace/mkdocs/home.md

@@ -0,0 +1,151 @@
+<style>
+/* These styles apply only to this page! */
+    .md-content__button {
+        display: none;
+    }
+  .md-typeset h1 {
+      line-height: 0;
+      margin: 0;
+      margin-left: -9999px;
+  }
+  .quickstart-wrapper {
+    min-width: 300px;
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+    padding-left: -50px;
+    column-gap: 50px;
+    row-gap: 50px;
+  }
+  .quickstart-wrapper > div {
+    flex: 300px;
+    max-width: 300px;
+  }
+  .tool-img{
+    box-shadow: rgba(0, 0, 0, 0.24) 0px 5px 5px;
+    border-radius: 5px;
+    min-width: 300px;
+    max-width: 300px;
+    max-height: 170px;
+    min-height: 170px;
+  }
+  .tool-caption{
+    font-family:  Roboto, Helvetica, sans-serif;
+    text-align: center;
+    margin-top: 10px;
+    font-size:  1.2rem;
+    font-weight: bold;
+
+    /* font-size: 1.25em;
+    font-weight: 400; */
+    letter-spacing: -.02em;
+    line-height: 1.5;
+  }
+  .tool-description{
+    font-family:  Helvetica, sans-serif;
+    text-align: center;
+    margin-top: 10px;
+    font-size:  0.7rem;
+    font-style: oblique;
+    /* font-weight: bold; */
+  }
+</style>
+
+
+{% 
+    set tools = [
+        {
+            "env": "IDE_URL",
+            "name": "IDE",
+            "image": "IDE.jpg",
+            "description": "Browser-based version of Visual Studio Code. Develop in any language, install hundreeds of extensions"
+        },
+        {
+            "env": "TERMINAL_URL",
+            "name": "Terminal",
+            "image": "Terminal.png",
+            "description": "Full-fledged browser-based terminal with Z-shell"
+        },
+        {
+            "env": "FILEBROWSER_URL",
+            "name": "File Browser",
+            "image": "Filebrowser.png",
+            "description": "Browse, upload and download files and folders to and from the Workspace"
+        },
+        {
+            "env": "CRONICLE_URL",
+            "name": "Cronicle",
+            "image": "Cronicle.jpg",
+            "description": "Schedule jobs, manage schedules, observe and monitor executions (user/pass - admin/admin)"
+        },
+        {
+            "env": "UNGIT_URL",
+            "name": "Ungit",
+            "image": "Ungit.jpg",
+            "description": "Manage Git repositories and work flow using beautiful UI"
+        },
+        {
+            "env": "STATICFS_URL",
+            "name": "Static File Server",
+            "image": "Static-server.png",
+            "description": "Serve any static websites like a breeze"
+        },
+        {
+            "env": "MC_URL",
+            "name": "M.Commander",
+            "image": "MC.jpg",
+            "description": "Feature rich visual file manager with internal text viewer and editor" 
+        },
+        {
+            "env": "HTOP_URL",
+            "name": "Process monitor",
+            "image": "Htop.jpg",
+            "description": "Monitor running process and resource utilization"
+        },
+        {
+            "env": "LUIGI",
+            "name": "Luigi",
+            "image": "Luigi.png",
+            "description": "Monitor pipelines and batch jobs"
+        },
+        {
+            "env": "JUPYTER",
+            "name": "Jupyter notebook",
+            "image": "Jupyter.png",
+            "description": "Jupyter notebook"
+        },
+        {
+            "env": "JUPYTERLAB",
+            "name": "Jupyterlab",
+            "image": "Jupyterlab.png",
+            "description": "Jupyter lab"
+        },
+        {
+            "env": "NBVIEWER",
+            "name": "Nbviewer",
+            "image": "Nbviewer.png",
+            "description": "Nbviewer"
+        }
+        ] 
+%}
+
+
+<div class="quickstart-wrapper">
+  {% for tool in tools %}
+    {% set tool_url = get_tool_url(tool.env) %}
+    <div>
+        <a href="{{ tool_url }}" target="_blank" rel="noopener noreferrer">
+            <img src="{{ tool.image }}" class="tool-img"/>
+        </a>
+        <a href="{{ tool_url }}">
+            <div class="tool-caption">{{ tool.name }}</div>
+        </a>
+        <div class="tool-description">{{ tool.description }}</div>
+    </div>
+  {% endfor %}
+</div>
+
+
+
+
+