rename infra-workspace to ansible-terraform. Add utils

This commit is contained in:
bluxmit 2021-08-29 09:28:32 +00:00
parent 5bccb02f86
commit 1f81939cbe
27 changed files with 1152 additions and 30 deletions

49
utils/ansible-report.sh Normal file
View file

@ -0,0 +1,49 @@
#!/bin/bash
while getopts i:o: flag
do
case "${flag}" in
i) inventory=${OPTARG};;
o) output_dir=${OPTARG};;
esac
done
if [ -z "${output_dir}" ]; then output_dir="/home/static-server/ansible-reports/"; fi
project_name=${PWD##*/}
printf '%s\n' "project name - ${project_name}"
report_dir="${output_dir}/${project_name}/$(date +"%d-%m-%Y-%H-%M-%S")"
mkdir -p "$report_dir"
# Ansible Lint
ansible-lint --nocolor > ${report_dir}/linter.txt
# ansible-cmdb
mkdir -p ./ansible-cmdb
if [ "${inventory}" ]; then
ansible -m setup -i ${inventory} --tree ./ansible-cmdb all
else
ansible -m setup --tree ./ansible-cmdb all
fi
ansible-cmdb ./ansible-cmdb > ${report_dir}/hosts.html
rm -rf ./ansible-cmdb
# Ansible inventory grapher
if [ "${inventory}" ]; then
ansible-inventory-grapher -i ${inventory} all | dot -Tpng > ${report_dir}/inventory.png
else
ansible-inventory-grapher all | dot -Tpng > ${report_dir}/inventory.png
fi
# Ansible playbook grapher
mkdir -p ${report_dir}/plays
if [ "${inventory}" ]; then
ls | grep yml | xargs -I{} ansible-playbook-grapher -i ${inventory} --include-role-tasks {} -o ${report_dir}/plays/{}
ls | grep yaml | xargs -I{} ansible-playbook-grapher -i ${inventory} --include-role-tasks {} -o ${report_dir}/plays/{}
else
ls | grep yml | xargs -I{} ansible-playbook-grapher --include-role-tasks {} -o ${report_dir}/plays/{}
ls | grep yaml | xargs -I{} ansible-playbook-grapher --include-role-tasks {} -o ${report_dir}/plays/{}
fi

View file

@ -21,7 +21,9 @@ port_increments = {
"IDE_URL": 5,
"TERMINAL_URL": 6,
"MC_URL": 7,
"HTOP_URL": 8
"HTOP_URL": 8,
"ANSIBLE_ARA": 9,
"TERRAFORM_BLAST_RADIUS": 10
}
workspace_meta = {
@ -40,6 +42,10 @@ workspace_meta = {
"mkdocs-magicspace": {
"port-range": 15,
"entrypoints": ["DOCS_URL", "FILEBROWSER_URL", "STATICFS_URL", "CRONICLE_URL", "UNGIT_URL", "IDE_URL", "TERMINAL_URL", "MC_URL", "HTOP_URL"]
},
"ansible-terraform-workspace": {
"port-range": 15,
"entrypoints": ["DOCS_URL", "FILEBROWSER_URL", "STATICFS_URL", "CRONICLE_URL", "UNGIT_URL", "IDE_URL", "TERMINAL_URL", "MC_URL", "HTOP_URL", "ANSIBLE_ARA", "TERRAFORM_BLAST_RADIUS"]
}
}
@ -151,7 +157,12 @@ def get_compose_dict(workspace_name, host_ip, start_port, user, password):
# Add Workspace values to the dict
y["services"]["workspace"] = {}
y["services"]["workspace"]["image"] = f"alnoda/{workspace_name}"
y["services"]["workspace"]["environment"] = {"WRK_HOST": host_ip, "WRK_PROTO": "https"}
y["services"]["workspace"]["environment"] = {
"WRK_HOST": host_ip,
"WRK_PROTO": "https",
"ARA_API_SERVER": f"http://{host_ip}:{start_port + port_increments["ANSIBLE_ARA"]}",
"ARA_API_CLIENT": "https"
}
y["services"]["workspace"]["labels"] = get_workspace_labels(ep)
# Add auth
authlabels = make_authlabels(user, password)
@ -186,7 +197,7 @@ def main(cmd_args):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--workspace", default="base-workspace")
parser.add_argument("--workspace", default="workspace-in-docker")
parser.add_argument("--port", default=8020)
parser.add_argument("--host")
parser.add_argument("--user")

27
utils/terraform-report.sh Normal file
View file

@ -0,0 +1,27 @@
#!/bin/bash
while getopts i:o: flag
do
case "${flag}" in
i) inventory=${OPTARG};;
o) output_dir=${OPTARG};;
esac
done
if [ -z "${output_dir}" ]; then output_dir="/home/static-server/terraform-reports/"; fi
project_name=${PWD##*/}
printf '%s\n' "project name - ${project_name}"
report_dir="${output_dir}/${project_name}/$(date +"%d-%m-%Y-%H-%M-%S")"
mkdir -p "$report_dir"
# terraform-docs
terrascan scan > ${report_dir}/terrascan.txt
# Terraform Graph
terraform graph | dot -Tsvg > .${report_dir}/graph.svg
# Terraform Visual
terraform plan -out=${report_dir}/plan.out
terraform show -json ${report_dir}/plan.out > ${report_dir}/plan.json
cd ${report_dir} && terraform-visual --plan ${report_dir}/plan.json

View file

@ -109,9 +109,8 @@
"wesbos.theme-cobalt2": "https://open-vsx.org/api/wesbos/theme-cobalt2/2.1.6/file/wesbos.theme-cobalt2-2.1.6.vsix",
"robbowen.synthwave-vscode": "https://open-vsx.org/api/RobbOwen/synthwave-vscode/0.1.8/file/RobbOwen.synthwave-vscode-0.1.8.vsix",
"vscoss.vscode-ansible": "https://open-vsx.org/api/vscoss/vscode-ansible/0.5.2/file/vscoss.vscode-ansible-0.5.2.vsix",
"dhoeric.ansible-vault": "https://open-vsx.org/api/dhoeric/ansible-vault/0.1.3/file/dhoeric.ansible-vault-0.1.3.vsix",
"hashicorp.terraform": "https://open-vsx.org/api/hashicorp/terraform/2.14.0/file/hashicorp.terraform-2.14.0.vsix",
"4ops.terraform": "https://open-vsx.org/api/4ops/terraform/0.2.1/file/4ops.terraform-0.2.1.vsix",
"samuelcolvin.jinjahtml": "https://open-vsx.org/api/samuelcolvin/jinjahtml/0.16.0/file/samuelcolvin.jinjahtml-0.16.0.vsix"
}
}

View file

@ -51,21 +51,33 @@ RUN rm -rf /opt/theia \
COPY --from=0 /opt/theia /opt/theia
COPY settings.json /home/abc/.theia/settings.json
COPY supervisord-infra.conf /etc/supervisord/
# Ara
COPY ara-settings.yaml /home/abc/.ara/server/settings.yaml
ENV ANSIBLE_CALLBACK_PLUGINS="$(python3 -m ara.setup.callback_plugins)" ARA_API_CLIENT="http" ARA_API_SERVER="http://0.0.0.0:8029" ARA_TIME_ZONE="UTC"
ENV ANSIBLE_CALLBACK_PLUGINS="/usr/local/lib/python3.9/dist-packages/ara/plugins/callback" ARA_API_CLIENT="http" ARA_API_SERVER="http://0.0.0.0:8029" ARA_TIME_ZONE="UTC"
RUN echo "------------------------------------------------------ ara" \
# Customize mkdocs
COPY ./mkdocs/mkdocs.yml /home/docs/mkdocs.yml
COPY ./mkdocs/home.md /home/docs/docs/pages/home
COPY ./mkdocs/Ara.png /home/docs/docs/pages/home/home/
COPY ./mkdocs/Blast-radius.png /home/docs/docs/pages/home/home/
COPY ./mkdocs/helpers.py /home/docs/macros
RUN echo "------------------------------------------------------ ansible ara" \
&& echo "UTC" > /etc/timezone \
&& mkdir -p /home/abc/.ara/server \
&& ara-manage makemigrations \
&& ara-manage migrate \
&& echo "------------------------------------------------------ ansible-lint" \
&& pip3 install "ansible-lint[yamllint]" \
&& echo "------------------------------------------------------ terraform" \
&& cd /tmp && wget https://releases.hashicorp.com/terraform/0.14.6/terraform_0.14.6_linux_amd64.zip \
&& unzip terraform_0.14.6_linux_amd64.zip -d /usr/local/bin \
&& rm /tmp/terraform_0.14.6_linux_amd64.zip \
&& echo "------------------------------------------------------ blast-radius" \
&& apt-get install -y graphviz \
&& mkdir -p /home/terraform \
&& echo "------------------------------------------------------ terraform-docs" \
&& cd /tmp && curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.15.0/terraform-docs-v0.15.0-linux-amd64.tar.gz \
&& tar -xzf terraform-docs.tar.gz \
@ -74,7 +86,7 @@ RUN echo "------------------------------------------------------ ara" \
&& rm /tmp/terraform-docs.tar.gz \
&& echo "------------------------------------------------------ tflint" \
&& curl -s https://raw.githubusercontent.com/terraform-linters/tflint/master/install_linux.sh | bash \
&& cd /tmp && curl -Lo /tmp/tfsec https://github.com/aquasecurity/tfsec/releases/download/v0.58.4/tfsec-linux-arm64
&& cd /tmp && curl -Lo /tmp/tfsec https://github.com/aquasecurity/tfsec/releases/download/v0.58.4/tfsec-linux-arm64 \
&& chmod +x /tmp/tfsec \
&& mv /tmp/tfsec /usr/bin/tfsec \
&& echo "------------------------------------------------------ terrascan" \
@ -82,7 +94,8 @@ RUN echo "------------------------------------------------------ ara" \
&& tar -xf terrascan.tar.gz terrascan && rm terrascan.tar.gz \
&& install terrascan /usr/local/bin && rm terrascan \
&& echo "------------------------------------------------------ terraform-visual" \
&& cd /tmp && curl -sL https://deb.nodesource.com/setup_12.x | -E bash - \
&& cd /tmp && curl -sL https://deb.nodesource.com/setup_12.x > tflintsetup.sh \
&& bash /tmp/tflintsetup.sh && rm /tmp/tflintsetup.sh \
&& apt-get update \
&& apt-get install -y nodejs \
&& apt-get install -y yarn \
@ -92,21 +105,24 @@ RUN echo "------------------------------------------------------ ara" \
&& tar -xzf terraform-inframap.tar.gz && rm terraform-inframap.tar.gz \
&& chmod +x inframap-linux-amd64 \
&& mv inframap-linux-amd64 /usr/bin/inframap \
&& echo "------------------------------------------------------ terraform rover" \
&& cd /tmp && curl -Lo ./terraform-rover.zip https://github.com/im2nguyen/rover/releases/download/v0.1.6/rover_0.1.6_linux_amd64.zip \
&& unzip /tmp/terraform-rover.zip -d /tmp/rover && rm terraform-rover.zip \
&& chmod +x /tmp/rover/rover_v0.1.6 \
&& mv /tmp/rover/rover_v0.1.6 /usr/bin/rover \
&& rm -rf /tmp/rover \
&& echo "------------------------------------------------------ mkdocs" \
&& echo "------------------------------------------------------ utils" \
&& rm -rf /home/abc/utils || true \
&& git clone https://github.com/bluxmit/alnoda-workspaces /tmp/alnoda-workspaces \
&& mv /tmp/alnoda-workspaces/utils /home/abc/ \
&& rm -rf /tmp/alnoda-workspaces \
&& echo "alias ansible-report='/home/abc/utils/ansible-report.sh'" >> /home/abc/.zshrc \
&& chmod +x /home/abc/utils/ansible-report.sh && chown abc /home/abc/utils/ansible-report.sh \
&& echo "alias terraform-report='/home/abc/utils/terraform-report.sh'" >> /home/abc/.zshrc \
&& chmod +x /home/abc/utils/terraform-report.sh && chown abc /home/abc/utils/terraform-report.sh \
&& echo "------------------------------------------------------ user" \
&& chown -R abc /home/abc/.ara/server \
&& chown -R abc /opt/theia \
&& mkdir -p /var/log/theia && chown -R abc /var/log/theia \
&& chown -R abc /var/log/ara/ \
&& mkdir -p /var/log/ara/ && chown -R abc /var/log/ara/ \
&& chown -R abc /home/docs \
&& chown -R abc /home/abc/utils \
&& chown -R abc /home/abc/installed-python-packages \
&& find /home -type d | xargs -I{} chown -R abc {} \
&& find /home -type f | xargs -I{} chown abc {}
USER abc

View file

@ -0,0 +1,541 @@
# Ansible-Terraform Workspace
This workspace - is a "dockerized" development environment with [Ansible](https://docs.ansible.com/)
and [Terraform](https://www.terraform.io/) installed, so that you don't need to do it yourself.
```
docker run --name space-1 -d -p 8020-8035:8020-8035 alnoda/ansible-terraform-workspace
```
and open [http://localhost:8020] in browser
## Contents
* [About](#about)
* [Use-cases](#use-cases)
* [Why documentation websites](#why-documentation-websites)
* [Features](#features)
* [The technology behind](#the-technology-behind)
* [Launch Workspace](#launch-workspace)
* [Workspace terminal](#workspace-terminal)
* [Multiple workspaces](#multiple-workspaces)
* [Open more ports](#open-more-ports)
* [Docker in docker](#docker-in-docker)
* [Run on remote server](#run-on-remote-server)
* [Use Workspace](#use-workspace)
* [Install applications](#install-applications)
* [Schedule jobs with Cron](#schedule-jobs-with-cron)
* [Python](#python)
* [Node.js](#node.js)
* [Run applications and services inside the workspace](#run-applications-and-services-inside-the-workspace)
* [Manage workspaces](#manage-workspaces)
* [Start and stop workspaces](#start-and-stop-workspaces)
* [Create new workspace image](#create-new-workspace-image)
* [Manage workspace images](#manage-workspace-images)
* [Save and load workspace images](#save-and-load-workspace-images)
* [Move workspace to the cloud](#move-workspace-to-the-cloud)
## About
Workspace containns browser-based Visual Studio Code, and other browser-based tools that make it comfortable working with dockerized
environnments.
[GIF]
## Use-cases
With the help of MkDocs-MagicSpace you can develop, build and serve the following kinds of websites:
1. Awesome looking docs for your open-source project. Use HTML & CSS to create front page and markdown for doc pages.
2. User manuals and API docs. While enterprise docs are for internal use, create great-looking API websites for your users, and manuals for your customers.
1. Enterprise documentation websites that unite numerous git repositories into one documentation project.
4. Tutorials and training websites. Do you have a lot of experience in something? Share it with the world.
MkDocs-MagicSpace helps you to highlight the code in many programming languages, write complex mathematical formulas,
[draw diagrams]((https://mermaid-js.github.io/mermaid/#/)) and so much more!
5. With MkDocs-MagicSpace you can also create books, booklets, and brochures as .pdf files.
One of the most convenient features of the MkDocs-MagicSpace - is that it is a workspace in docker.
You use it on your local machine, you can run it on the remote server and collaborate with your colleagues,
you can give it to your peer as a whole. And you can use it to serve your docs from any server, no matter
if it is public or inside the company's VPN.
## Why documentation websites
*Why create separate documentation websites? Why not just write Git readme files?*
- MkDocs website with beautiful themes looks much better than any readme file. The resulting documentation website looks professional and awesome.
- MkDocs adds text search to your documentation website.
- In the case of closed-source software, sharing readme files from the git repository with external users is not an option.
- Github does not render beautiful extended markdown features like admonitions, tabs, etc. Neither renders diagrams, formulas, swagger docs, or notebooks.
- Using MkDocs-MagicSpace you create documentation from the same markdown readme files you have in your repo together with the code. And you
can create a unified documentation website from multiple repositories in Github, GitLab, Bitbucket.
- You can add such features as Google Analytics, multi-language localization.
*And what about Confluent?*
The approach of having docs in different places (Git repositories, Confluent, Wiki, etc.) has serious drawbacks. It creates chaos, and documentation inevitably diverges with time.
And you never know where to look for the information you need.
The best docs live together with the code. With MkDocs-MagicSpace you can follow this practice with ease, build a documentation website
from one or many git repositories, include readme files that live close to the code. You can have a build process that even creates
auto-documentation directly from the code.
## Features
MkDocs-MagicSpace has [**MkDocs**](https://squidfunk.github.io/mkdocs-material/) installed with a collection of extensions and plugins
that bring MkDocs to the next level.
MkDocs-MagicSpace is an extension of the [workspace-in-docker](https://github.com/Alnoda/workspaces-in-docker/blob/main/workspaces/workspace-in-docker/README.md)
and has all its features:
- **Workspace UI** - launch all workspace tools from one place.
- [**Eclipse Theia**](https://theia-ide.org/docs/) - open source version of popular Visual Studio Code IDE. Theia is trully open-source, has
VS-Code extensions and works in browser. This means it can run inside a docker container on local machine or in cloud.
- [**FileBrowser**](https://github.com/filebrowser/filebrowser) - manage files and folders inside the workspace, and exchange data between local environment and the workspace
- [**Cronicle**](https://github.com/jhuckaby/Cronicle) - task scheduler and runner, with a web based front-end UI. It handles both scheduled, repeating and on-demand jobs, targeting any number of worker servers, with real-time stats and live log viewer.
- [**Static File Server**](https://github.com/vercel/serve) - view any static html sites as easy as if you do it on your local machine. Serve static websites easily.
- [**Ungit**](https://github.com/FredrikNoren/ungit) - rings user friendliness to git without sacrificing the versatility of it.
- [**Midnight Commander**](https://midnight-commander.org/) - Feature rich visual file manager with internal text viewer and editor.
- [**Process Monitor**](https://htop.dev/) - Monitor running process and resource utilization.
Built on top of Base-workspace and Ubuntu-workspace, this workspace gets all the features those workspaces have.
In particular, workspace-in-docker provides excellent experience when working directly in the terminal, and has docker-in-docker.
#### The technology behind
MkDocs-MagicSpace in its essense is all about [**MkDocs**](https://www.mkdocs.org/) - a fast, simple and downright gorgeous static site generator that's geared towards
building project documentation.
MkDocs has many themes, MkDocs-MagicSpace has [**Material for MkDocs**](https://squidfunk.github.io/mkdocs-material/) set up and configured by default.
It is one of the best themes for MkDocs, which makes your website look very professional. This theme is extremely customizable,
searchable, mobile-friendly, has 40+ languages, has built-in search. It also adds lots of markdown features such as tabbed content containers, mathematical formulas, critic markup,
task lists, and more than 10k icons and emojis.
MkDocs-MagicSpace has lots of packages and extensions already installed:
- [PyMdown Extensions](https://facelessuser.github.io/pymdown-extensions/) - add even more cool features of the extended markdown: sub- and superscripts, keys, magic links, sane headers etc.
- [Mkdocs-macro plugin](https://mkdocs-macros-plugin.readthedocs.io/en/latest/) - add variables and macros written in Python!
- [Mkdocs-monorepo plugin](https://backstage.github.io/mkdocs-monorepo-plugin/) - build multiple documentation folders in a single Mkdocs. Designed for large codebases.
- [MkDocs Newsletter](https://lyz-code.github.io/mkdocs-newsletter/) - show the changes of documentation repositories in a user friendly format, at the same time that it's easy for the authors to maintain.
- [Mkdocs-mermaid2-plugin](https://github.com/fralau/mkdocs-mermaid2-plugin) - renders textual graph descriptions into Mermaid graphs (flow charts, sequence diagrams, pie charts, etc.).
- [Pygments](https://pygments.org/) - a generic syntax highlighter suitable for use in code hosting, forums, wikis or other applications that need to prettify source code, with over 500 languages and other text formats.
- [Mkdocs-include-markdown-plugin](https://github.com/mondeja/mkdocs-include-markdown-plugin) - include Markdown files completely or partially, and include files of any type.
- [Mkdocs-table-reader-plugin]() - directly insert CSV files as tables in your website.
**(The complete list of installed packages - /home/abc/installed-python-packages/mkdocs-requirements.txt)**
## Launch Workspace
Workspaces - are merely docker containers, that's why managing workspaces is easy and intuitive - it is enough to know only docker commands,
no need to learn any new tools.
In order to avoid confusion, the following convention is adopted:
```sh
command to execute outside of the workspace
```
> `command to execute inside the workspace (after entering running docker container)`
To start a workspace simply execute in terminal
```sh
docker run --name space-1 -d -p 8020-8035:8020-8035 alnoda/mkdocs-magicspace
```
*(It is recommended to run workspace in the daemon mode)*
***Open [http://localhost:8020](http://localhost:8020)***
Workspace has its own UI, which includes quiklaunch (home) page and documentation pages.
From the quiklaunch you can open any workspace tool. Documentation pages you modify in order
to document the project, workspace use and setup.
### Workspace terminal
There are several ways how to work with terminal of the the mkdocs-magicspace:
- built-it in-browser terminal
- use terminal provided by in-browser IDE [http://localhost:8025](http://localhost:8025) ([unless other ports are mapped](#multiple-workspaces))
- ssh into the running the docker container (of the workspace) from your terminal
<p align="center">
<img src="https://raw.githubusercontent.com/bluxmit/alnoda-workspaces/main/workspaces/base-workspace/img/base-workspace-terminal.gif" alt="Base-Workspace terminal" width="500">
</p>
*(Browser-based terminals always work under the user you started the workspace with, the default is non root user "abc")*
If you want to enter running workspace container from your terminal execute:
```sh
docker exec -it space-1 /bin/zsh
```
If you don't want to use z-shell
```
docker exec -it space-1 /bin/bash
```
This way allows to ssh into the workspace as a root user at any time, even if the workspace itself was not starter as root user (the default user is abc)
```sh
docker exec -it --user=root space-1 /bin/zsh
```
You can work in Ubuntu terminal now. Execute the followinng command to know your workspace user
> `whoami`
### Multiple workspaces
Every workspace requires range of ports. If one workspace is up and running, the ports 8020-8035 are taken.
mkdocs-magicspace itself uses 9 ports (8020-8028), but it is recommended to map several extra ports just in case. Having extra ports,
you can always launch new applications on these ports, and they will be immediately exposed outside of the workspace.
In order to start another workspace, you either need to stop currently runnning workspace, or to run another workspace
on the different port range.
If you are planning to run more than one workspace at the same time, you can run another workspace with
the different port range, for example
```sh
docker run --name space-2 -d -p 8040-8055:8020-8035 -e ENTRY_PORT=8040 alnoda/mkdocs-magicspace
```
Notice that in addition we need to set environmental variable ENTRY_PORT, which should be equal to the first port in the new range.
Workspace UI usues this variable to know the new port range, and redirects to the proper addresses of the workspace applications' UIs.
### Open more ports
We started workspace container with a port range mapped "-p 8020-8035". If you are planning to expose more applications
from inside of a container, add additional port mapping, for example
```sh
docker run --name space-1 -d -p 8020-8035:8020-8035 -p 8080:8080 alnoda/mkdocs-magicspace
```
You can add multiple port mappings:
```sh
docker run --name space-1 -d -p 8020-8035:8020-8035 -p 8080:8080 -p 443:443 alnoda/mkdocs-magicspace
```
**NOTE:** It is not a problem if you don't expose any ports, but later on realise you need them -
you will just create new image, and run it exposing the required port (look in the section [Create new image](#create-new-image))
### Docker in docker
It is possible to work with docker directly from the workspace (using workspace terminal).
```
docker run --name space-1 -d -p 8020-8035:8020-8035 -v /var/run/docker.sock:/var/run/docker.sock alnoda/mkdocs-magicspace
```
NOTE: in order to use docker in docker you need to or enter into the workspace container as root
```sh
docker exec -it --user=root space-1 /bin/zsh
```
### Run on remote server
Because workspace is just a docker image, running it in any other server is as easy as running it on local laptop.
Running on remote server makes it much simpler to collaborate, because you can just share credentials to the workspace with your peers, and they will be able to use it.
You can also run applications that should run permanently, and run jobs on schedule.
#### Unsecure remote workspace
The simplest deployment of the workkspace requires only 3 steps:
- get virtual server on your favourite cloud (Digital Ocean, Linode, AWS, GC, Azure ...)
- [install docker](https://docs.docker.com/engine/install/) on this server
- ssh to the remote server and start workspace
```
docker run --name space-1 -d -p 8020-8035:8020-8035 -e WRK_HOST="<ip-of-your-remote-server>" alnoda/mkdocs-magicspace
```
**NOTE:** When running workspace on the remote server, add envronmental variable `-e WRK_HOST="<ip-of-your-remote-server>"`.
Workspace UI needss this variable to know how redirect properly to the workspace applications' UIs.
Open in your browser `<ip-of-your-remote-server>:8020`
If docker-in-docker is required, then
```
docker run --name space-1 -d -p 8020-8035:8020-8035 -e WRK_HOST="<ip-of-your-remote-server>" -v /var/run/docker.sock:/var/run/docker.sock alnoda/mkdocs-magicspace
```
This way launches workspace in cloud, but such workspace is not secure, everyone who knows IP of your server will be able to use it.
#### Secure remote workspace
*You might want to restrict access to the workspace, and secure encrypted communication with the workspace*
Kkdocs-MagicSpace contains utility that will generate everything needed to launch the workspace in cloud in a secure way, with authentication and with TLS.
If you want to run workspace on the remote server securely, start mkdocs-magicspace on your local laptop first, open its terminal and
use utility `/home/abc/utils/remote.py` to generate create docker-compose project with TLS certificates. Simply execute
> `python /home/abc/utils/remote.py --workspace="mkdocs-magicspace" --port="8020" --host="68.183.69.198" --user="user1" --password="pass1"`
**NOTE:** you have to specify the correct host (IP of the server you want to run the workspace on), and user and password of your choice.
You see folder `/home/abc/utils/remote` is created. Copy this folder to the remote server (any location). Ssh to the server, cd into
the directory you copied and execute `docker-compose up -d`.
That's it, you workspace is running securely on the remote server, using
self-signed TLS certificates for encrypted https communication between you laptop and the remote workspace,
and authentication is added.
## Use Workspace
Among the common actions you'd do in the workspace are
- installation of new applications and runtimes
- edit files, write code, scripts
- build, compile and execute code
- start/stop applications and services
- schedule tasks and scripts
- process data
### Install applications
Use workspace workspace terminal to install new applications.
Install with ```sudo apt install```. The default *abc* user is allowed to install packages.
For example, in order to install [Emacs text editor](https://www.gnu.org/software/emacs/) open workspace terminal, and execute
> `sudo apt install emacs`
### Schedule jobs with Cron
Schedule execution of any task with cron - a time-based job scheduler in Unix-like computer operating systems.
Open workspace terminal, and execute
> `crontab -e`
*(chose [1] nano as editor on the first time)*
In the end of the opened file add line
> `* * * * * echo $(whoami) >> /home/cron.txt`
This will print every minute username to file */home/cron.txt* . *(Hit Ctrl+X to exit nano)*
Hint: example of cron job definition:
```
.---------------- minute (0 - 59)
| .------------- hour (0 - 23)
| | .---------- day of month (1 - 31)
| | | .------- month (1 - 12) OR jan,feb,mar,apr ...
| | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
| | | | |
* * * * * command to be executed
```
**NOTE** you can disconnect from the image and close terminal - cron will continue working.
> Instead of cron you might want to use Cronicle - a tool with Web UI, and a great list of features
> that will provide you with the dashboard, list of executions and statistics, even let you ser limis
> on resources for each jobs, and create depenndencies between jobs.
### Python
Python and Pip are installed. To use python console, open workspace terminal and execute
> `python`
install python package with pip, for
> `pip install pandas`
If you are planning to work with python, we recommend to install IPython, that provides a rich toolkit to help
you make the most of using Python interactively. Install and start ipython
> ```pip install ipython```
> `ipython`
### Node.js
We recommend to use nodeenv to create different node environments.
For example, open workspace terminal, create folder npmgui, and activate environment with node v. 12.18.3 and npm v.6.0.0
> `cd /home`
> `mkdir npmgui; cd npmgui`
> `nodeenv --node=12.18.3 --npm=6.0.0 env`
Let's install package and start node application
> `. env/bin/activate && npm i -g npm-gui`
> `npm-gui 0.0.0.0:8030`
Open your browser on http://localhost:8030/
**NOTE:** If you close terminal, the application will stop. See how to [start applications that reamin live after closing a workspace terminal](#run-applications-and-services-inside-the-workspace)
### Run applications and services inside the workspace
If you want application to keep running after workspace terminal is closed start it with **"&!"** at the end.
For example, in the last section we started *npm-gui* tool with command `npm-gui 0.0.0.0:8030`. If you close the workspace terminal,
this application witll stop running. To keep it running after terminal is closed, execute
> `npm-gui 0.0.0.0:8030 &!`
Now, if you disconnect from the workspace and close terminal, the application will continue running in the workspace, untill [workspace is stopped](#start-and-stop-workspaces).
## Manage workspaces
Workspace is just a docker container. You can start, stop, delete and do anything you can do with docker images and containers.
There are two concepts to keep in mind: **images** and **containers**. Images are workspace blueprints. For example, **alnoda/mkdocs-magicspace** -
is an image. When you execute this command
```sh
docker run --name space-1 -d -p 8020-8035:8020-8035 alnoda/mkdocs-magicspace
```
you create container called **space-1** from the image **alnoda/mkdocs-magicspace**. You can create any number of containers, but you need to
[map different ports to each of them](#multiple-workspaces).
Container - is your workspace. You can start, stop and delete them. You can run multiple workspace containers at the same time, or work with
one workspace at a time.
From the workspace (which is a container) you can create new image. This is called **commit docker image**.
Essentially, this means *"take my workspace and create new image with all the changes I've done in my workspace*"
### Start and stop workspaces
The workspace started in daemon mode will continue working in the background.
See all the running docker containers
```
docker ps
```
Stop workspace
```sh
docker stop space-1
```
Workspace is stopped. All the processes and cron jobs are not running.
See all docker conntainers, including stopped
```
docker ps -a
```
Start workspace again. Processes and cron jobs are resumed.
```sh
docker start space-1
```
Delete workspace container (all work will be lost)
```
docker rm space-1
```
### Create new workspace image
Having made changes, you can commit them creating new image of the workspace. In order to create new workspace image with the
name "space-image" and version "0.2" execute
```
docker commit space-1 space-image:0.2
```
Run new workspace with
```
docker run --name space2 -d space-image:0.2
```
The new workspace accommodates all the changes that you've made in your space-1. Hence you can have versions of your workspaces.
Create different versions before the important changes.
### Manage workspace images
See all docker images
```
docker images
```
Delete workspace image entirely
```
docker rmi -f alnoda/mkdocs-magicspace
```
**NOTE:** you cannot delete image if there is a running container created from it. Stop container first.
### Save and load workspace images
After you commit workspace container, and create new image out of it, you can push it to your docker registry or save it in a file.
#### Save workspace in a file
Assuming you created new image **space-image:0.4** from your workspace, you can save it as a tar file
```
docker save space-image:0.4 > space-image-0.4.tar
```
We can delete the image with
```
docker rmi -f space-image:0.4
```
And restore it from the tar file
```
docker load < space-image-0.4.tar
```
#### Push workspace to a registry
A better way to manage images is docker registries. You can use docker registries in multiple clouds. They are cheap annd very convenient.
Check out for example, [Registry in DigitalOcean](https://www.digitalocean.com/products/container-registry/) or in [Scaleway container registry](https://www.scaleway.com/en/container-registry/). There are more.
Pushing image to registry is merely 2 extra commands: 1) tag image; 2) push image
You will be able to pull image on any device, local or cloud.
### Move workspace to the cloud
Ease of running workspace in cloud, and ability to move workspaces between local machine and remote server -
is one of the main features of the workspace, and the reasonn why the workspace is entirely in docker.
It is often a case that experiment, which started on personal notebook require more computational
resources, must be running for a long period of time, or executed periodically. All of these cases are
the reasons to move a workspace to the cloud server. Usually it is a hassle, but this workspace can be moved
to the remote server easily.
The easiest way to move workspace to the cloud is to get your private docker registry. Then moving a workspace from a laptop to
a remote server is only 3 commands:
1. [Commit workspace to the a image](#create-new-workspace-image)
2. [Push workspace to your docker registry](https://docs.docker.com/engine/reference/commandline/push/)
3. ssh to remote server, and [run workspace there](#run-on-remote-server)
If you don't want to use container registry, then there are 2 steps more involved:
1. [Commit workspace to the a image](#create-new-workspace-image)
2. [Save image to file](save-and-loa-images)
3. Copy file to remote server. There are many options:
- Launch filexchange workspace on the remote server
- Use [cyberduck](https://cyberduck.io/)
- use [scp](https://linuxize.com/post/how-to-use-scp-command-to-securely-transfer-files/)
4. [Load workspace image from file](#save-and-load-workspace-images) on the remote server
5. [Start workspace on the remote server](#run-on-remote-server)

View file

@ -0,0 +1,53 @@
# Ansible tools
### [Terraform inventory](https://github.com/adammck/terraform-inventory)
### [Ansible-cmdb](https://github.com/fboender/ansible-cmdb)
Ansible-cmdb takes the output of Ansible's fact gathering and converts
it into a static HTML overview page (and other things) containing system configuration information.
```
ansible -m setup --tree out/ all
ansible-cmdb out/ > overview.html
```
### [Ansible inventory grapher](https://github.com/willthames/ansible-inventory-grapher)
```
ansible-inventory-grapher all | dot -Tpng > /home/static-server/my.png
```
### [Ansible Playbook Grapher](https://github.com/haidaraM/ansible-playbook-grapher)
```
ansible-playbook-grapher --include-role-tasks example.yml -o /home/static-server/example
```
### [Ansible Lint](https://ansible-lint.readthedocs.io/en/latest/installing.html)
```
ansible-lint --nocolor > /home/static-server/ansible-lint.txt
```
### Ansible Mitogen
Read [how to configure your Ansible project to work 7x faster!](https://mitogen.networkgenomics.com/ansible_detailed.html#installation)
### [Ansible Doctor](https://ansible-doctor.geekdocs.de/)
ansible-doctor is a simple annotation like documentation generator based on Jinja2 templates.
hile ansible-doctor comes with a default template called readme, it is also possible to write your own templates.
The first step is to identify if the given folder is an Ansible role. This check is very simple,
if the folder contains a sub-directory called tasks is MUST be an Ansible role! :)
After the successful check, ansible-doctor will try to read some static files into a dictionary:
- defaults/main.yml
- meta/main.yml
```
ansible-doctor -o /home/static-server/ roles/example_production
```

View file

@ -0,0 +1,13 @@
repos:
- repo: git://github.com/antonbabenko/pre-commit-terraform
rev: v1.50.0 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
hooks:
- id: terraform_fmt
- id: terraform_docs
- id: terraform_validate
- id: terraform_docs_without_aggregate_type_defaults
- id: terraform_docs_replace
- id: terraform_tflint
- id: terraform_tfsec
- id: checkov
- id: terrascan

View file

@ -1,8 +1,4 @@
## Ansible tools
## Terraform tools
# Terraform tools
### Pre-commit hook
@ -50,7 +46,30 @@ inframap generate terraform.tfstate | dot -Tpng > graph.png
```
rover --workingDir /home/terraform
```
NOTE: Rover serves only on port 9000
NOTE: Rover serves only on port 9000
UPDATE: made fork and configured to serve on any port
ISSUES: does not work, no resources displayed
In order to add Rover to the Workspace:
- Include in Dockerfile
```
cd /tmp && curl -Lo ./terraform-rover.zip https://github.com/bluxmit/rover/releases/download/v1.6-cust/w1.7.zip \
&& unzip /tmp/terraform-rover.zip -d /tmp/rover && rm terraform-rover.zip \
&& chmod +x /tmp/rover/rover \
&& mv /tmp/rover/rover /usr/bin/rover \
&& rm -rf /tmp/rover \
```
- Include in supervisord-infra.conf
```
[program:rover]
directory=/home/terraform
command=/bin/sh -c " rover --workingDir /home/terraform "
```
## Tutorials

View file

@ -0,0 +1,13 @@
repos:
- repo: git://github.com/antonbabenko/pre-commit-terraform
rev: v1.50.0 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
hooks:
- id: terraform_fmt
- id: terraform_docs
- id: terraform_validate
- id: terraform_docs_without_aggregate_type_defaults
- id: terraform_docs_replace
- id: terraform_tflint
- id: terraform_tfsec
- id: checkov
- id: terrascan

View file

@ -0,0 +1,33 @@
# Create Server in Scaleway with Terraform
## Prepare workspace
Set your credentials as environment variable in your workspace.
Add the following lines to /home/abc/.zshrc:
```
export SCW_DEFAULT_PROJECT_ID=<YOUR_PROJECT_ID>
export SCW_ACCESS_KEY=<YOUR_ACCESS_KEY>
export SCW_SECRET_KEY=<YOUR_SECRET_KEY>
```
## Terraform
- Initialize terraform project
```
terraform init
```
- Show Terraform plan
```
terraform plan
```
- Apply to create new ifrastructure
```
terraform apply
```
- Destroy infrastructure completely
```
terraform destroy
```

View file

@ -0,0 +1,50 @@
terraform {
required_providers {
scaleway = {
source = "scaleway/scaleway"
}
}
required_version = ">= 0.13"
}
provider "scaleway" {
zone = "fr-par-1"
}
resource "scaleway_instance_ip" "public_ip" {}
resource "scaleway_instance_server" "web" {
type = "DEV1-S"
image = "ubuntu_focal"
ip_id = scaleway_instance_ip.public_ip.id
security_group_id = scaleway_instance_security_group.web.id
root_volume {
delete_on_termination = false
}
additional_volume_ids = [ scaleway_instance_volume.data.id ]
}
resource "scaleway_instance_security_group" "web" {
name = "http"
description = "allow HTTP and HTTPS traffic"
inbound_rule {
action = "accept"
port = 80
ip_range = "0.0.0.0/0"
protocol = "TCP"
}
inbound_rule {
action = "accept"
port = 443
ip_range = "0.0.0.0/0"
protocol = "TCP"
}
}
resource "scaleway_instance_volume" "data" {
size_in_gb = 100
type = "b_ssd"
}

View file

@ -1,9 +1,16 @@
## Ansible tools
ansible==4.4.0
cffi==1.14.6
# https://github.com/fboender/ansible-cmdb
ansible-cmdb==1.31
# https://github.com/ansible-community/ara
ara==1.5.7
# https://github.com/willthames/ansible-inventory-grapher
ansible-inventory-grapher==2.5.0
# https://github.com/haidaraM/ansible-playbook-grapher
ansible-playbook-grapher==0.11.1
# https://ansible-doctor.geekdocs.de/
ansible-doctor==0.3.1
## Terraform tools
# https://github.com/28mm/blast-radius

Binary file not shown.

After

Width:  |  Height:  |  Size: 284 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 342 KiB

View file

@ -0,0 +1,59 @@
"""
Basic example of a Mkdocs-macros module.
Include this {{ macros_info() }} in any page to get complete macro info
"""
import os
port_increments = {
"DOCS_URL": 0,
"FILEBROWSER_URL": 1,
"STATICFS_URL": 2,
"CRONICLE_URL": 3,
"UNGIT_URL": 4,
"IDE_URL": 5,
"TERMINAL_URL": 6,
"MC_URL": 7,
"HTOP_URL": 8,
"ANSIBLE_ARA": 9,
"TERRAFORM_BLAST_RADIUS": 10
}
# this function name should not be changed
def define_env(env):
"""
This is the hook for defining variables, macros and filters
- variables: the dictionary that contains the environment variables
- macro: a decorator function, to declare a macro.
- filter: a function with one of more arguments,
used to perform a transformation
"""
@env.macro
def get_tool_url(env):
try:
return os.environ[name]
except:
# Get host
host = "localhost"
try:
host = os.environ["WRK_HOST"]
except:
pass
proto = "http"
try:
proto = os.environ["WRK_PROTO"]
except:
pass
# Entry port - port relative to which other ports will be calculated
entry_port = 8020
try:
entry_port = int(os.environ["ENTRY_PORT"])
except:
pass
# Assign port
try:
port = port_increments[env] + entry_port
except:
port = 80
return f"{proto}://{host}:{port}"

View file

@ -0,0 +1,139 @@
<style>
/* These styles apply only to this page! */
.md-content__button {
display: none;
}
.md-typeset h1 {
line-height: 0;
margin: 0;
margin-left: -9999px;
}
.quickstart-wrapper {
min-width: 300px;
display: flex;
flex-wrap: wrap;
justify-content: center;
padding-left: -50px;
column-gap: 50px;
row-gap: 50px;
}
.quickstart-wrapper > div {
flex: 300px;
max-width: 300px;
}
.tool-img{
box-shadow: rgba(0, 0, 0, 0.24) 0px 5px 5px;
border-radius: 5px;
min-width: 300px;
max-width: 300px;
max-height: 170px;
min-height: 170px;
}
.tool-caption{
font-family: Roboto, Helvetica, sans-serif;
text-align: center;
margin-top: 10px;
font-size: 1.2rem;
font-weight: bold;
/* font-size: 1.25em;
font-weight: 400; */
letter-spacing: -.02em;
line-height: 1.5;
}
.tool-description{
font-family: Helvetica, sans-serif;
text-align: center;
margin-top: 10px;
font-size: 0.7rem;
font-style: oblique;
/* font-weight: bold; */
}
</style>
{%
set tools = [
{
"env": "IDE_URL",
"name": "IDE",
"image": "IDE.jpg",
"description": "Browser-based version of Visual Studio Code. Develop in any language, install hundreeds of extensions"
},
{
"env": "TERMINAL_URL",
"name": "Terminal",
"image": "Terminal.png",
"description": "Full-fledged browser-based terminal with Z-shell"
},
{
"env": "FILEBROWSER_URL",
"name": "File Browser",
"image": "Filebrowser.png",
"description": "Browse, upload and download files and folders to and from the Workspace"
},
{
"env": "CRONICLE_URL",
"name": "Cronicle",
"image": "Cronicle.jpg",
"description": "Schedule jobs, manage schedules, observe and monitor executions (user/pass - admin/admin)"
},
{
"env": "UNGIT_URL",
"name": "Ungit",
"image": "Ungit.jpg",
"description": "Manage Git repositories and work flow using beautiful UI"
},
{
"env": "STATICFS_URL",
"name": "Static File Server",
"image": "Static-server.png",
"description": "Serve any static websites like a breeze"
},
{
"env": "MC_URL",
"name": "M.Commander",
"image": "MC.jpg",
"description": "Feature rich visual file manager with internal text viewer and editor"
},
{
"env": "HTOP_URL",
"name": "Process monitor",
"image": "Htop.jpg",
"description": "Monitor running process and resource utilization"
},
{
"env": "ANSIBLE_ARA",
"name": "Ansible Ara",
"image": "Ara.png",
"description": "Monitor for all Ansible plays"
},
{
"env": "TERRAFORM_BLAST_RADIUS",
"name": "Terraform Blast Radius",
"image": "Blast-radius.png",
"description": "Visualise terraform dependency graphf for a project in folder /home/terraform"
}
]
%}
<div class="quickstart-wrapper">
{% for tool in tools %}
{% set tool_url = get_tool_url(tool.env) %}
<div>
<a href="{{ tool_url }}" target="_blank" rel="noopener noreferrer">
<img src="{{ tool.image }}" class="tool-img"/>
</a>
<a href="{{ tool_url }}">
<div class="tool-caption">{{ tool.name }}</div>
</a>
<div class="tool-description">{{ tool.description }}</div>
</div>
{% endfor %}
</div>

View file

@ -0,0 +1,70 @@
# ===========================================================
# NAVIGATION
# ===========================================================
nav:
- Home: pages/home/home.md
- About: README.md
- Docs: docs.md
# ===========================================================
# CONFIGURATION
# ===========================================================
site_name: Infra Workspace
repo_url: https://github.com/bluxmit/alnoda-workspaces
site_url: https://alnoda.org
edit_uri: ""
# ===========================================================
# APPEARANCE
# ===========================================================
theme:
name: 'material'
favicon: 'assets/favicon.ico'
logo: 'assets/Alnoda-logo.svg'
custom_dir: overrides
icon:
repo: fontawesome/brands/git-alt
features:
- navigation.instant
palette:
- scheme: default
toggle:
icon: material/toggle-switch-off-outline
name: Switch to light mode
primary: red
accent: orange
- scheme: slate
toggle:
icon: material/toggle-switch
name: Switch to dark mode
primary: red
accent: orange
extra:
# Link to open when your logo is clicked
homepage: https://alnoda.org
host_url: http://localhost
plugins:
- search
# Enable Macros and jinja2 templates
- macros:
module_name: macros/helpers
extra_javascript:
- javascripts/config.js
- https://polyfill.io/v3/polyfill.min.js?features=es6

View file

@ -8,7 +8,3 @@ logfile_maxbytes = 1024
[program:blast-radius]
directory=/home/terraform
command=/bin/sh -c " blast-radius --serve --port 8030 /home/terraform "
[program:rover]
directory=/home/terraform
command=/bin/sh -c " rover --workingDir /home/terraform "

View file

@ -0,0 +1,13 @@
ARG docker_registry=docker.io/alnoda
ARG image_tag=18.04-0.5
FROM ${docker_registry}/workspace-in-docker:${image_tag}
USER root
ENV PATH=$PATH:/opt/go/bin GOPATH=/home/project/
RUN cd /tmp && wget https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz \
&& tar -C /opt -xzf go1.15.2.linux-amd64.tar.gz
USER abc

View file

@ -0,0 +1,13 @@
ARG docker_registry=docker.io/alnoda
ARG image_tag=18.04-0.5
FROM ${docker_registry}/workspace-in-docker:${image_tag}
USER root
ENV PATH=$PATH:/opt/go/bin GOPATH=/home/project/
RUN cd /tmp && wget https://golang.org/dl/go1.17.linux-amd64.tar.gz \
&& tar -C /opt -xzf go1.17.linux-amd64.tar.gz
USER abc

View file

@ -3,12 +3,12 @@
MkDocs-MagicSpace is an all-in-one tool, carefully crafted to develop, build and serve awesome static websites, for the purpose
of documentation, tutorials, and training.
#### Try it out
```
docker run --name space-1 -d -p 8020-8035:8020-8035 alnoda/mkdocs-magicspace
```
and open [http://localhost:8020] in browser
## Contents
* [Use-cases](#use-cases)

View file

@ -13,6 +13,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& chmod 0777 /usr/local/bin/systemctl \
&& apt-get install -y software-properties-common \
&& apt-get install -y apache2-utils \
&& apt-get install -y zip gzip tar \
&& echo "------------------------------------------------------ User" \
&& useradd -u 8877 abc \
&& chmod -R 777 /home \