documentation improvements (#261)

* adds documentation

* updates in user docs

* documentation

Co-authored-by: styiannis <1515939+styiannis@users.noreply.github.com>

Dockerfile

@@ -29,8 +29,8 @@ FROM python:3.8-slim-buster as runtime-image
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV ADMIN_USER='admin'
-ENV ADMIN_PASSWORD='mediacms'
 ENV ADMIN_EMAIL='admin@localhost'
+#ENV ADMIN_PASSWORD='uncomment_and_set_password_here'
 
 # See: https://github.com/celery/celery/issues/6285#issuecomment-715316219
 ENV CELERY_APP='cms'
@@ -50,11 +50,17 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io
 
 RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends \
-    supervisor nginx ffmpeg imagemagick procps -y && \
+    supervisor nginx imagemagick procps wget xz-utils -y && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get purge --auto-remove && \
     apt-get clean
 
+RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \
+    mkdir -p tmp && \
+    tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp && \
+    cp -v tmp/ffmpeg tmp/ffprobe tmp/qt-faststart /usr/local/bin && \
+    rm -rf tmp ffmpeg-release-amd64-static.tar.xz 
+
 WORKDIR /home/mediacms.io/mediacms
 
 EXPOSE 9000 80

README.md

@@ -4,7 +4,7 @@
 [![Code Quality: Cpp](https://img.shields.io/lgtm/grade/javascript/g/mediacms-io/mediacms.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/mediacms-io/mediacms/context:javascript)
 <br/>
 [![GitHub license](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://raw.githubusercontent.com/mediacms-io/mediacms/main/LICENSE.txt)
-[![Downloads](https://img.shields.io/badge/download-all%20releases-brightgreen.svg)](https://github.com/mediacms-io/mediacms/releases/)
+[![Releases](https://img.shields.io/github/v/release/mediacms-io/mediacms?color=green)](https://github.com/mediacms-io/mediacms/releases/)
 [![DockerHub](https://img.shields.io/docker/pulls/mediacms/mediacms)](https://hub.docker.com/repository/docker/mediacms/mediacms/)
 
 
@@ -83,6 +83,11 @@ For a small to medium installation, with a few hours of video uploaded daily, an
 In terms of disk space, think of what the needs will be. A general rule is to multiply by three the size of the expected uploaded videos (since the system keeps original versions, encoded versions plus HLS), so if you receive 1G of videos daily and maintain all of them, you should consider a 1T disk across a year (1G * 3 * 365).
 
 
+## Releases
+
+Visit [Releases Page](https://github.com/mediacms-io/mediacms/releases) for detailed Changelog
+
+
 ## Installation / Maintanance
 
 There are two ways to run MediaCMS, through Docker Compose and through installing it on a server via an automation script that installs and configures all needed services. Find the related pages:
@@ -92,7 +97,13 @@ There are two ways to run MediaCMS, through Docker Compose and through installin
 
 ## Configuration
 
-Several options are available on cms/settings.py, most of the things that are allowed or should be disallowed are described there. It is advisable to override any of them by adding it to cms/local_settings.py. All configuration options will be documented gradually on the [Configuration](docs/Configuration.md) page.
+Visit [Configuration](docs/Configuration.md) page.
+
+
+## Documentation
+* [Users documentation](docs/user_docs.md) page
+* [Administrators documentation](docs/admins_docs.md) page
+* [Developers documentation](docs/developers_docs.md) page
 
 
 ## Technology
@@ -117,12 +128,6 @@ If you like the project, here's a few things you can do
 - Star the project
 - Add functionality, work on a PR, fix an issue! 
 
-## Developers info
-
-- API documentation through Swagger is available under /swagger URL of your installation -example https://demo.mediacms.io/swagger/
-- We're working on proper documentation for users, managers and developers, until then checkout what's available on the docs/ folder of this repository
-- Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code.
-- Checkout the [Code of conduct page](CODE_OF_CONDUCT.md) if you want to contribute to this repository
 
 ## Contact
 info@mediacms.io

deploy/docker/prestart.sh

@@ -4,20 +4,29 @@ RANDOM_ADMIN_PASS=`python -c "import secrets;chars = 'abcdefghijklmnopqrstuvwxyz
 ADMIN_PASSWORD=${ADMIN_PASSWORD:-$RANDOM_ADMIN_PASS}
 
 if [ X"$ENABLE_MIGRATIONS" = X"yes" ]; then
+    echo "Running migrations service"
     python manage.py migrate
-    python manage.py loaddata fixtures/encoding_profiles.json
-    python manage.py loaddata fixtures/categories.json
+    EXISTING_INSTALLATION=`echo "from users.models import User; print(User.objects.exists())" |python manage.py shell`
+    if [ "$EXISTING_INSTALLATION" = "True" ]; then 
+        echo "Loaddata has already run"
+    else
+        echo "Running loaddata and creating admin user"
+        python manage.py loaddata fixtures/encoding_profiles.json
+        python manage.py loaddata fixtures/categories.json
+
+    	# post_save, needs redis to succeed (ie. migrate depends on redis)
+        DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
+            --no-input \
+            --username=admin \
+            --email=$ADMIN_EMAIL \
+            --database=default || true
+        echo "Created admin user with password: $ADMIN_PASSWORD"
+
+    fi
+    echo "RUNNING COLLECTSTATIC"
+
     python manage.py collectstatic --noinput
 
-    echo "Admin Password: $ADMIN_PASSWORD"
-
-    # post_save, needs redis to succeed (ie. migrate depends on redis)
-    DJANGO_SUPERUSER_PASSWORD=$ADMIN_PASSWORD python manage.py createsuperuser \
-        --no-input \
-        --username=$ADMIN_USER \
-        --email=$ADMIN_EMAIL \
-        --database=default || true
-
     # echo "Updating hostname ..."
     # TODO: Get the FRONTEND_HOST from cms/local_settings.py
     # echo "from django.contrib.sites.models import Site; Site.objects.update(name='$FRONTEND_HOST', domain='$FRONTEND_HOST')" | python manage.py shell

docker-compose-http-proxy.yaml

@@ -18,6 +18,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-https-proxy.yaml

@@ -20,6 +20,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-letsencrypt.yaml

@@ -38,6 +38,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose-named-volumes.yaml

@@ -11,6 +11,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docker-compose.yaml

@@ -11,6 +11,8 @@ services:
       ENABLE_CELERY_SHORT: 'no'
       ENABLE_CELERY_LONG: 'no'
       ENABLE_CELERY_BEAT: 'no'
+    command: "./deploy/docker/prestart.sh"
+    restart: on-failure
     depends_on:
       redis:
         condition: service_healthy

docs/Configuration.md

@@ -1,6 +1,6 @@
 ## Configuration
 
-A number of options are available on `cms/settings.py`. 
+Several options are available on `cms/settings.py`, most of the things that are allowed or should be disallowed are described there.
 
 It is advisable to override any of them by adding it to `local_settings.py` . 
 

docs/Docker_Compose.md

@@ -30,6 +30,13 @@ docker-compose up
 
 This will download all MediaCMS related Docker images and start all containers. Once it finishes, MediaCMS will be installed and available on http://localhost or http://ip
 
+A user admin has been created with random password, you should be able to see it at the end of migrations container, eg
+
+```
+migrations_1     | Created admin user with password: gwg1clfkwf
+```
+
+or if you have set the ADMIN_PASSWORD variable on Dockerfile, that variable will be set as the admin user's password
 
 ## Update
 

docs/admins_docs.md

@@ -0,0 +1,29 @@
+# Administrators documentation
+
+## Table of contents
+- [Manage pages](#manage-pages)
+- [Django admin dashboard](#django-admin-dashboard)
+- [On portal workflow](#on-portal-workflow)
+- [On user roles](#on-user-roles)
+- [Adding languages for Captions and subtitles](#adding-languages-for-captions-and-subtitles)
+- [Add/delete categories and tags](#adddelete-categories-and-tags)
+- [Video transcoding](#video-transcoding)
+
+
+## Manage pages
+
+## Django admin dashboard
+
+## On portal workflow
+Who can publish content, how content appears on public listings.Difference between statuses (private, unlisted, public)
+
+## On user roles
+Differences over MediaCMS manager, MediaCMS editor, logged in user
+
+## Adding languages for Captions and subtitles
+
+## Add/delete categories and tags
+Through the admin section - http://your_installation/admin/
+
+## Video transcoding
+Add / remove resolutions and profiles through http://your_installation/admin/encodeprofile

docs/developers_docs.md

@@ -0,0 +1,83 @@
+# Developers documentation
+
+## Table of contents
+- [System architecture](#system-architecture)
+- [API documentation](#api-documentation)
+- [How to contribute](#how-to-contribute)
+- [Working with Docker tips](#working-with-docker-tips)
+- [How video is transcoded](#how-video-is-transcoded)
+
+## How to contribute
+Before you send a PR, make sure your code is properly formatted. For that, use `pre-commit install` to install a pre-commit hook and run `pre-commit run --all` and fix everything before you commit. This pre-commit will check for your code lint everytime you commit a code.
+
+Checkout the [Code of conduct page](../CODE_OF_CONDUCT.md) if you want to contribute to this repository
+
+
+## System architecture
+
+## API documentation
+API is documented using Swagger - checkout ot http://your_installation/swagger - example https://demo.mediacms.io/swagger/
+This page allows you to login to perform authenticated actions - it will also use your session if logged in. 
+
+
+## Working with Docker tips
+
+To perform the Docker installation, follow instructions to install Docker + Docker compose (docs/Docker_Compose.md) and then build/start docker-compose-dev.yaml . This will run the frontend application on port 8088 on top of all other containers (including the Django web application on port 80)
+
+```
+docker-compose -f docker-compose-dev.yaml build
+docker-compose -f docker-compose-dev.yaml up
+```
+
+### Frontend application changes
+Eg change `frontend/src/static/js/pages/HomePage.tsx` , dev application refreshes in a number of seconds (hot reloading) and I see the changes, once I'm happy I can run
+
+```
+docker-compose -f docker-compose-dev.yaml -T frontend npm run dist
+```
+
+And then in order for the changes to be visible on the application while served through nginx, 
+
+```
+cp -r frontend/dist/static/* static/
+```
+
+POST calls: cannot be performed through the dev server, you have to make through the normal application (port 80) and then see changes on the dev application on port 8088. 
+Make sure the urls are set on `frontend/.env` if different than localhost
+
+
+Media page: need to upload content through the main application (nginx/port 80), and then use an id for page media.html, for example `http://localhost:8088/media.html?m=nc9rotyWP`
+
+There are some issues with CORS too to resolve, in order for some pages to function, eg the manage comments page
+
+```http://localhost:8088/manage-media.html px manage_media
+```
+
+### Backend application changes
+After I make changes to the django application (eg make a change on `files/forms.py`) in order to see the changes I have to restart the web container
+
+```
+docker-compose -f docker-compose-dev.yaml restart web
+```
+
+## How video is transcoded
+
+Original files get uploaded to the application server, and they get stored there as FileFields.
+
+If files are videos and the duration is greater than a number (defined on settings, I think 4minutes), they are also broken in chunks, so one Encode object per chunk, for all enabled EncodeProfiles.
+
+Then the workers start picking Encode objects and they transcode the chunks, so if a chunk gets transcoded correctly, the original file (the small chunk) gets replaced by the transcoded file, and the Encode object status is marked as 'success'.
+
+
+original.mp4 (1G, 720px)--> Encode1 (100MB, 240px, chunk=True), Encode2 (100MB, 240px, chunk=True)...EncodeXX (100MB, 720px, chunk=True) ---> when all Encode objects are success, for a resolution, they get concatenated to the original_resolution.mp4 file and this gets stored as Encode object (chunk=False). This is what is available for download.
+
+Apparently the Encode object is used to store Encoded files that are served eventually (chunk=False, status='success'), but also files while they are on their way to get transcoded (chunk=True, status='pending/etc')
+
+(Parenthesis opening)
+there is also an experimental small service (not commited to the repo currently) that speaks only through API and a) gets tasks to run, b) returns results. So it makes a request and receives an ffmpeg command, plus a file, it runs the ffmpeg command, and returns the result.I've used this mechanism on a number of installations to migrate existing videos through more servers/cpu and has worked with only one problem, some temporary files needed to be removed from the servers (through a periodic task, not so big problem)
+(Parenthesis closing)
+
+
+When the Encode object is marked as success and chunk=False, and thus is available for download/stream, there is a task that gets started and saves an HLS version of the file (1 mp4-->x number of small .ts chunks). This would be FILES_C
+
+This mechanism allows for workers that have access on the same filesystem (either localhost, or through a shared network filesystem, eg NFS/EFS) to work on the same time and produce results. 

docs/user_docs.md

@@ -0,0 +1,208 @@
+# Users documentation
+
+## Table of contents
+- [Uploading media](#uploading-media)
+- [Downloading media](#downloading-media)
+- [Adding captions/subtitles](#adding-captionssubtitles)
+- [Search media](#search-media)
+- [Share media](#share-media)
+- [Embed media](#embed-media)
+- [Customize my profile options](#customize-my-profile-options)
+
+## Uploading media
+
+### How to Upload Media
+
+Uploading media is as simple as clicking the _Upload Media_ button, waiting for media to upload, and then clicking the media to add metadata (title, description etc.) by filling out a form.
+
+#### Click Upload Button
+
+Click the _Upload Media_ button from the right-side of the screen at the top:
+
+<p align="left">
+    <img src="./images/Click-Upload-Media-button.png"/>
+</p>
+
+#### Upload Page
+
+Clicking the _Upload Media_ button takes you to the upload page at a URL like this:
+
+https://demo.mediacms.io/upload
+
+#### Click Browse Button
+
+Here you should click the _"Browse your files"_ button (or drag and drop a file from your desktop):
+
+<p align="left">
+    <img src="./images/Click-Browse-button.png"/>
+</p>
+
+#### Select media file and click Open button
+
+Select the media file that you want to upload and click the _"Open"_ button:
+
+<p align="left">
+    <img src="./images/Select-Media-File-Click-Open.png"/>
+</p>
+
+#### Wait for file to upload
+
+Wait for the file to finish uploading:
+
+<p align="left">
+    <img src="./images/Uploading.png"/>
+</p>
+
+#### Pause uploading
+
+If you want you can pause upload by clicking _Pause button_:
+
+<p align="left">
+    <img src="./images/Pause-button.png"/>
+</p>
+
+#### Continue uploading
+
+Continue upload by clicking _Continue button_:
+
+<p align="left">
+    <img src="./images/Continue-button.png"/>
+</p>
+
+#### Wait for media to finish Processing
+
+Wait for the media file to finish Processing:
+
+<p align="left">
+    <img src="./images/Processing.png"/>
+</p>
+
+#### Click View media button
+
+Click the View media button to open the media page:
+
+<p align="left">
+    <img src="./images/Click-View-media-button.png"/>
+</p>
+
+#### Media will be in the encoding queue
+
+The media will take some time to finish encoding (MediaCMS will transcode the file to several formats and resolutions). Meanwhile you can edit the media file to add metadata.
+
+#### Click Edit Media button
+
+Click the EDIT MEDIA button to add metadata and configure the poster image:
+
+<p align="left">
+    <img src="./images/Click-Edit-Media-button.png"/>
+</p>
+
+#### Add Metadata (Summary, Description etc.)
+
+Make sure you fill in all the required fields, and try to complete as many of the non-required fields as possible. This ensures the database is populated with useful meta-data to help others access useful information about you and your video.
+
+<p align="left">
+    <img src="./images/Edit-Media-Metadata-1.png"/>
+</p>
+
+<p align="left">
+    <img src="./images/Edit-Media-Metadata-2.png"/>
+</p>
+
+
+
+## Downloading media
+
+MediaCMS offers a configurable option whereby users can make their media files available for download. Downloads are available for transcoded files, and the original file.
+
+#### How To Enable Download
+
+Visit the media view page and choose the EDIT MEDIA button.
+
+Select the checkbox for "Allow Downloads"
+
+#### How To Download Media
+
+Visit the media view page and click the DOWNLOAD button below the video player.
+
+<p align="left">
+    <img src="./images/Click-Download-Button.png">
+</p>
+
+Choose the version you wish to download - a transcoded file or the original file:
+
+<p align="left">
+    <img src="./images/Click-version-download.png">
+</p>
+
+Choose Save File and click the OK button.
+
+<p align="left">
+    <img src="./images/Save-File.png">
+</p>
+
+
+
+## Adding captions/subtitles
+
+With MediaCMS you can add subtitles/captions to your video by uploading a subtitles file in the popular .vtt format.
+
+### Visit Media Page & Click EDIT SUBTITLE Button
+
+Visit the "single media page" for the media you wish to add subtitles/captions to and click the EDIT SUBTITLES button:
+
+<p align="left">
+    <img src="./images/Click-EDIT-SUBTITLE.png"/>
+</p>
+
+### Upload Subtitles in .vtt Format
+
+Click the Language menu to select the correct language for your Subtitles/Captions file:
+
+<p align="left">
+    <img src="./images/Click-Subtitle-Language-Menu.png"/>
+</p>
+
+Choose the correct Language for your file:
+
+<p align="left">
+    <img src="./images/Subtitles-captions1.png"/>
+</p>
+
+Click Browse to find a subtitles/captions file on your computer (if your file is not in the .vtt format, you may find a conversion service on the Internet):
+
+<p align="left">
+    <img src="./images/Subtitles-captions2.png"/>
+</p>
+
+Choose a .vtt subtitles/captions file from your computer:
+
+<p align="left">
+    <img src="./images/Subtitles-captions3.png"/>
+</p>
+
+Click the Add button to add the file to your media:
+
+<p align="left">
+    <img src="./images/Click-ADD-button.png"/>
+</p>
+
+### View Subtitles/Captions in Video Player
+
+You can now watch the captions/subtitles play back in the video player - and toggle display on/off by clicking the CC button:
+
+<p align="left">
+    <img src="./images/CC-display.png"/>
+</p>
+
+## Search media
+How search can be used
+
+## Share media
+How to share media
+
+## Embed media
+How to use the embed media option
+
+## Customize my profile options
+Customize profile and channel

install.sh

@@ -21,21 +21,24 @@ It is expected to run on a new system **with no running instances of any these s
 done
 
 
-if [[ `lsb_release -d` == *"Ubuntu 20"* ]]; then
+osVersion=$(lsb_release -d)
+if [[ $osVersion == *"Ubuntu 20"* ]] || [[ $osVersion == *"Ubuntu 18"* ]] || [[ $osVersion == *"buster"* ]]; then
     echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
-elif [[ `lsb_release -d`  = *"Ubuntu 18"* ]]; then
-    echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
-# added check for Debian 10 (buster)
-elif [[ `lsb_release -d` == *"buster"* ]]; then
-    echo 'Performing system update and dependency installation, this will take a few minutes'
-    apt-get update && apt-get -y upgrade && apt install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip ffmpeg imagemagick python3-certbot-nginx certbot wget -y
+    apt-get update && apt-get -y upgrade && apt-get install python3-venv python3-dev virtualenv redis-server postgresql nginx git gcc vim unzip imagemagick python3-certbot-nginx certbot wget xz-utils -y
 else
     echo "This script is tested for Ubuntu 18 and 20 versions only, if you want to try MediaCMS on another system you have to perform the manual installation"
-exit
+    exit
 fi
 
+# install ffmpeg
+echo "Downloading and installing ffmpeg"
+wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+mkdir -p tmp
+tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C tmp
+cp -v tmp/{ffmpeg,ffprobe,qt-faststart} /usr/local/bin
+rm -rf tmp ffmpeg-release-amd64-static.tar.xz
+echo "ffmpeg installed to /usr/local/bin"
+
 read -p "Enter portal URL, or press enter for localhost : " FRONTEND_HOST
 read -p "Enter portal name, or press enter for 'MediaCMS : " PORTAL_NAME