Database Upgrades

* Wrote a new module that allows database upgrades via the admin dashboard.
* Added a new table t_version that allows to check the version of the database.
* The upgrade path is for vanilla databases. It exports and backups data, then reintegrates it.

.gitignore

@@ -7,3 +7,4 @@ db/*.db-wal
 output
 /output/
 *.old
+/backup/

labertasche.yaml

@@ -12,23 +12,10 @@ system:
   database_uri: "sqlite:///db/labertasche.db"                           # Database URI. See documentation. Default is sqlite.
   secret: "6Gxvb52bIJCm2vfDsmWKzShKp1omrzVG"                            # CHANGE ME! THIS IS IMPORTANT!
   output: "./__implementation_example/data/"                            # Base path for the output json
-  debug: false                                                          # Leave this as is, this is for development.
+  debug: true                                                           # Leave this as is, this is for development.
   send_otp_to_publish: true                                             # Disables confirmation w/ OTP via mail
   cookie_secure: false
 
-projects:
-  - default:
-      web_url: "http://dev.localhost:1314/"
-      blog_url: "http://dev.localhost:1313/"
-      output: "./__implementation_example/data/"
-      send_otp_to_publish: true
-
-  - example.com:
-      web_url: "http://comments.example.com/"
-      blog_url: "http://blog.example.com/"
-      output: "./example/data/"
-      send_otp_to_publish: true
-
 gravatar:
   cache: true                                                           # Enable caching of gravatar images
   static_dir: "./__implementation_example/static/images/gravatar/"      # Where to store cached images, must exist!

labertasche/blueprints/__init__.py

@@ -10,3 +10,4 @@ from .bp_comments import bp_comments
 from .bp_login import bp_login
 from .bp_dashboard import bp_dashboard
 from .bp_jsconnector import bp_jsconnector
+from .bp_upgrades import bp_dbupgrades

labertasche/blueprints/bp_upgrades/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# /**********************************************************************************
+#  * _author  : Domeniko Gentner
+#  * _mail    : code@tuxstash.de
+#  * _repo    : https://git.tuxstash.de/gothseidank/labertasche
+#  * _license : This project is under MIT License
+#  *********************************************************************************/
+from flask import Blueprint
+
+# Blueprint
+bp_dbupgrades = Blueprint("bp_dbupgrades", __name__, url_prefix='/upgrade')
+
+from .db_v2 import upgrade_db_to_v2

labertasche/blueprints/bp_upgrades/db_v2.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# /**********************************************************************************
+#  * _author  : Domeniko Gentner
+#  * _mail    : code@tuxstash.de
+#  * _repo    : https://git.tuxstash.de/gothseidank/labertasche
+#  * _license : This project is under MIT License
+#  *********************************************************************************/
+from . import bp_dbupgrades
+from flask_cors import cross_origin
+from flask_login import login_required
+from flask import render_template, jsonify, make_response
+from pathlib import Path
+from labertasche.database import labertasche_db as db
+from labertasche.models import TProjects, TComments, TLocation, TEmail, TVersion
+from labertasche.helper import Settings
+from json import dump, load
+from shutil import copy, make_archive
+from re import search
+from secrets import compare_digest
+from datetime import datetime
+
+
+def get_backup_folder() -> Path:
+    path = Path('.').absolute() / "backup" / "v1"
+    return path
+
+
+@cross_origin()
+@bp_dbupgrades.route('/db_v2/')
+@login_required
+def upgrade_db_to_v2():
+    # TODO: Check if db has already been upgraded
+    status = False
+    try:
+        version = db.session.query(TVersion).first()
+        if version:
+            status = True
+    except Exception as e:
+        print(e.__class__)
+        pass
+
+    return render_template("db-upgrades.html", title="DB upgrade V1 to V2",
+                           prev_version=1, new_version=2, status=status)
+
+
+@cross_origin()
+@bp_dbupgrades.route('/db_v2/backup/', methods=['GET'])
+@login_required
+def upgrade_db_to_v2_backup():
+    path = get_backup_folder()
+    # Create path for backup
+    try:
+        if not path.exists():
+            path.mkdir(mode=755, exist_ok=True, parents=True)
+    except OSError as e:
+        return make_response(jsonify(status='exception', msg=str(e)), 400)
+
+    return make_response(jsonify(status="ok"), 200)
+
+
+@cross_origin()
+@bp_dbupgrades.route('/db_v2/export/')
+@login_required
+def upgrade_db_to_v2_export():
+    path = get_backup_folder()
+
+    # make sure nothing is pending
+    db.session.commit()
+
+    # Export tables
+    t_locations = db.session.query(TLocation.id_location, TLocation.location).all()
+    t_emails = db.session.query(TEmail.id_email, TEmail.email, TEmail.is_allowed, TEmail.is_blocked).all()
+    t_comments = db.session.query(TComments.comments_id, TComments.location_id, TComments.email,
+                                  TComments.content, TComments.created_on, TComments.is_published,
+                                  TComments.is_spam, TComments.spam_score, TComments.replied_to,
+                                  TComments.confirmation, TComments.deletion, TComments.gravatar).all()
+
+    locations = []
+    for loc in t_locations:
+        locations.append({
+            "id_location": loc.id_location,
+            "location": loc.location
+        })
+
+    emails = []
+    for mail in t_emails:
+        emails.append({
+            "id_email": mail.id_email,
+            "email": mail.email,
+            "is_allowed": mail.is_allowed,
+            "is_blocked": mail.is_blocked
+        })
+
+    comments = []
+    for comment in t_comments:
+        comments.append({
+            "comments_id": comment.comments_id,
+            "location_id": comment.location_id,
+            "email": comment.email,
+            "content": comment.content,
+            "created_on": f"{comment.created_on.__str__()}",
+            "is_published": comment.is_published,
+            "is_spam": comment.is_spam,
+            "spam_score": comment.spam_score,
+            "replied_to": comment.replied_to,
+            "confirmation": comment.confirmation,
+            "deletion": comment.deletion,
+            "gravatar": comment.gravatar
+        })
+
+    # Output jsons
+    try:
+        p_export_location = path / "locations.json"
+        with p_export_location.open('w') as fp:
+            dump(locations, fp, indent=4, sort_keys=True)
+
+        p_export_mail = path / "emails.json"
+        with p_export_mail.open('w') as fp:
+            dump(emails, fp, indent=4, sort_keys=True)
+
+        p_export_comments = path / "comments.json"
+        with p_export_comments.open('w') as fp:
+            dump(comments, fp, indent=4, sort_keys=True)
+
+    except Exception as e:
+        return make_response(jsonify(status='exception-write-json', msg=str(e)), 400)
+
+    # Copy database
+    try:
+        settings = Settings()
+        db_uri = settings.system['database_uri']
+        if compare_digest(db_uri[0:6], "sqlite"):
+            m = search("([/]{3})(.*)", db_uri)
+            new_db = get_backup_folder() / "labertasche.db"
+            old_db = Path(m.group(2)).absolute()
+            copy(old_db, new_db)
+    except Exception as e:
+        return make_response(jsonify(status='exception-copy-db', msg=str(e)), 400)
+
+    make_archive(path, "zip", path)
+
+    return make_response(jsonify(status='ok'), 200)
+
+
+@cross_origin()
+@bp_dbupgrades.route('/db_v2/recreate/')
+@login_required
+def upgrade_db_to_v2_recreate():
+    try:
+        db.drop_all()
+        db.session.flush()
+        db.session.commit()
+        db.create_all()
+    except Exception as e:
+        return make_response(jsonify(status='exception', msg=str(e)), 400)
+
+    return make_response(jsonify(status='ok'), 200)
+
+
+@cross_origin()
+@bp_dbupgrades.route('/db_v2/import/')
+@login_required
+def upgrade_db_to_v2_import():
+    path = get_backup_folder()
+    settings = Settings()
+
+    try:
+        # load location
+        p_loc = (path / 'locations.json').absolute()
+        with p_loc.open('r') as fp:
+            locations = load(fp)
+
+        # load mails
+        m_loc = (path / 'emails.json').absolute()
+        with m_loc.open('r') as fp:
+            mails = load(fp)
+
+        # load comments
+        c_loc = (path / 'comments.json').absolute()
+        with c_loc.open('r') as fp:
+            comments = load(fp)
+
+    except FileNotFoundError as e:
+        return make_response(jsonify(status='exception-filenotfound', msg=str(e)), 400)
+
+    # Create project
+    default_project = {
+        "id_project": 1,
+        "name": "default",
+        "weburl": settings.system['web_url'],
+        "blogurl": settings.system['blog_url'],
+        "output": settings.system['output'],
+        "sendotp": settings.system['send_otp_to_publish'],
+        "gravatar_cache": settings.gravatar['cache'],
+        "gravatar_cache_dir": settings.gravatar['static_dir'],
+        "gravatar_size": settings.gravatar['size'],
+        "addon_smileys": settings.addons['smileys']
+    }
+
+    # Create db version, so we can track it in the future
+    version = {
+        "id_version": 1,
+        "version": 2
+    }
+
+    try:
+        # Add to db
+        db.session.add(TVersion(**version))
+        db.session.add(TProjects(**default_project))
+
+        # walk json and readd to database with project set to project 1
+        for each in mails:
+            each.update({'project_id': 1})
+            db.session.add(TEmail(**each))
+
+        for each in locations:
+            each.update({'project_id': 1})
+            db.session.add(TLocation(**each))
+
+        for each in comments:
+            each.update({'project_id': 1})
+            dt = datetime.fromisoformat(each['created_on'])
+            each.update({'created_on': dt})
+            db.session.add(TComments(**each))
+
+        # Commit
+        db.session.commit()
+        db.session.flush()
+    except Exception as e:
+        return make_response(jsonify(status='exception-database', msg=str(e)), 400)
+
+    return make_response(jsonify(status='ok'), 200)

labertasche/database/__init__.py

@@ -13,7 +13,7 @@ from sqlalchemy import MetaData
 convention = {
     "ix": 'ix_%(column_0_label)s',
     "uq": "uq_%(table_name)s_%(column_0_name)s",
-    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "ck": "ck_%(table_name)s_%(column_0_name)s",
     "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
     "pk": "pk_%(table_name)s"
 }

labertasche/models/__init__.py

@@ -10,3 +10,4 @@ from .t_comments import TComments
 from .t_location import TLocation
 from .t_emails import TEmail
 from .t_projects import TProjects
+from .t_version import TVersion

labertasche/models/t_version.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# /**********************************************************************************
+#  * _author  : Domeniko Gentner
+#  * _mail    : code@tuxstash.de
+#  * _repo    : https://git.tuxstash.de/gothseidank/labertasche
+#  * _license : This project is under MIT License
+#  *********************************************************************************/
+from labertasche.database import labertasche_db as db
+
+
+class TVersion(db.Model):
+    # table name
+    __tablename__ = "t_version"
+    __table_args__ = {'useexisting': True}
+
+    # primary key
+    id_version = db.Column(db.Integer, primary_key=True, autoincrement=True)
+
+    # data
+    version = db.Column(db.Integer)

labertasche/settings/__init__.py

@@ -28,4 +28,4 @@ class Settings:
         self.gravatar = conf['gravatar']
         self.addons = conf['addons']
         self.smileys = conf['smileys']
-        self.projects = conf['projects']
+

server.py

@@ -9,17 +9,15 @@
 import logging
 from flask import Flask, redirect, url_for
 from flask_cors import CORS
-from sqlalchemy import event
+from sqlalchemy import event, inspect
 # noinspection PyProtectedMember
 from sqlalchemy.engine import Engine
 from labertasche.settings import Settings
 from labertasche.database import labertasche_db
-from labertasche.blueprints import bp_comments, bp_login, bp_dashboard, bp_jsconnector
+from labertasche.blueprints import bp_comments, bp_login, bp_dashboard, bp_jsconnector, bp_dbupgrades
-from labertasche.models import TProjects
 from labertasche.helper import User
 from flask_login import LoginManager
-from flask_migrate import Migrate
+from datetime import timedelta
-
 
 # Load settings
 settings = Settings()
@@ -30,48 +28,48 @@ laberflask.config.update(dict(
     SESSION_COOKIE_DOMAIN=settings.system['cookie_domain'],
     SESSION_COOKIE_SECURE=settings.system['cookie_secure'],
     REMEMBER_COOKIE_SECURE=settings.system['cookie_secure'],
+    REMEMBER_COOKIE_DURATION=timedelta(days=7),
+    REMEMBER_COOKIE_HTTPONLY=True,
+    REMEMBER_COOKIE_REFRESH_EACH_REQUEST=True,
     DEBUG=settings.system['debug'],
     SECRET_KEY=settings.system['secret'],
-    TEMPLATES_AUTO_RELOAD=True,
+    TEMPLATES_AUTO_RELOAD=settings.system['debug'],
     SQLALCHEMY_DATABASE_URI=settings.system['database_uri'],
     SQLALCHEMY_TRACK_MODIFICATIONS=False
 ))
 
-# Flask migrate
-migrate = Migrate(laberflask, labertasche_db, render_as_batch=True)
-
-# Initialize ORM
-labertasche_db.init_app(laberflask)
-with laberflask.app_context():
-    labertasche_db.create_all()
-    project = labertasche_db.session.query(TProjects).filter(TProjects.id_project == 1).first()
-    if not project:
-        default_project = {
-            "id_project": 1,
-            "name": "default"
-        }
-        labertasche_db.session.add(TProjects(**default_project))
-        labertasche_db.session.commit()
-
 # CORS
-CORS(laberflask, resources={r"/comments": {"origins": settings.system['blog_url']}})
+CORS(laberflask, resources={r"/comments": {"origins": settings.system['blog_url']},
+                            r"/api": {"origins": settings.system['web_url']},
+                            r"/dashboard": {"origins": settings.system['web_url']},
+                            })
 
 # Import blueprints
 laberflask.register_blueprint(bp_comments)
 laberflask.register_blueprint(bp_dashboard)
 laberflask.register_blueprint(bp_login)
 laberflask.register_blueprint(bp_jsconnector)
+laberflask.register_blueprint(bp_dbupgrades)
 
 # Disable Werkzeug's verbosity during development
 log = logging.getLogger('werkzeug')
 log.setLevel(logging.ERROR)
 
-
 # Set up login manager
 loginmgr = LoginManager(laberflask)
 loginmgr.login_view = 'bp_admin_login.login'
 
+# Initialize ORM
+labertasche_db.init_app(laberflask)
+with laberflask.app_context():
+    table_names = inspect(labertasche_db.get_engine()).get_table_names()
+    is_empty = table_names == []
+    # Only create tables if the db is empty, so we can a controlled upgrade.
+    if is_empty:
+        labertasche_db.create_all()
+
 
+# There is only one user
 @loginmgr.user_loader
 def user_loader(user_id):
     if user_id != "0":
@@ -79,11 +77,13 @@ def user_loader(user_id):
     return User(user_id)
 
 
+# User not authorized
 @loginmgr.unauthorized_handler
 def login_invalid():
     return redirect(url_for('bp_login.show_login'))
 
 
+# Enable write-ahead-log for sqlite databases
 @event.listens_for(Engine, "connect")
 def set_sqlite_pragma(dbapi_connection, connection_record):
     if settings.system["database_uri"][0:6] == 'sqlite':

templates/base.html

@@ -4,15 +4,16 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=yes">
     <meta name="description" content="labertasche comment system dashboard">
-    <link rel="preconnect" href="https://cdn.materialdesignicons.com">
 
     <!-- Preload -->
     <link rel="preload" href="/static/css/open-sans-v18-latin-regular.woff2" as="font" type="font/woff2" crossorigin="anonymous" media="all">
-    <link rel="preload" as="style" href="/static/css/labertasche.css">
+    <link rel="preload" href="/static/css/materialdesignicons-webfont.woff2" as="font" type="font/woff2" crossorigin="anonymous" media="all">
-    <link rel="preload" href="https://cdn.materialdesignicons.com/5.4.55/css/materialdesignicons.min.css" as="style">
+    <link rel="preload" href="/static/css/labertasche.css" as="style" crossorigin="anonymous" media="screen">
+    <link rel="preload" href="/static/css/materialdesignicons.min.css" as="style" crossorigin="anonymous" media="screen">
+    <link rel="preload" href="/static/css/Chart.min.css" as="style" media="screen">
 
-    <link rel="stylesheet" href="https://cdn.materialdesignicons.com/5.4.55/css/materialdesignicons.min.css">
     <link rel="stylesheet" href="/static/css/labertasche.css" media="screen">
+    <link rel="stylesheet" href="/static/css/materialdesignicons.min.css" media="screen">
     <link rel="stylesheet" href="/static/css/Chart.min.css" media="screen">
 
     <title>labertasche Dashboard</title>
@@ -49,21 +50,22 @@
         {% block main %}
         {% endblock %}
         </section>
-<script defer src="/static/js/dashboard.js"></script>
+        <script defer src="/static/js/dashboard.js"></script>
-<script defer src="/static/js/Chart.bundle.min.js"></script>
+        <script defer src="/static/js/Chart.bundle.min.js"></script>
-<script defer src="https://unpkg.com/@popperjs/core@2"></script>
+        <script defer src="/static/js/popper.min.js"></script>
-<script defer src="https://unpkg.com/tippy.js@6"></script>
+        <script defer src="/static/js/tippy-bundle.umd.min.js"></script>
-<script defer>
+        {% block javascript_libs %}
-        document.addEventListener('DOMContentLoaded', () => {
-
-        // Comments
-        const urlParams = new URLSearchParams(window.location.search);
-        if (urlParams.get("error") === "404"){
-            show_modal('modal-project-not-found');
-        }
-        {% block javascript %}
         {% endblock %}
-        });
+        <script defer>
-</script>
+                document.addEventListener('DOMContentLoaded', () => {
+
+                const urlParams = new URLSearchParams(window.location.search);
+                if (urlParams.get("error") === "404"){
+                    show_modal('modal-project-not-found');
+                }
+                {% block javascript %}
+                {% endblock %}
+                });
+        </script>
 </body>
 </html>

templates/db-upgrades.html

@@ -0,0 +1,31 @@
+{% extends "base.html" %}
+{% block main %}
+<div style="min-height: 80vh;" class="container bg-deepmatte p-6 brdr-yayellow is-size-5">
+    <h1 class="is-size-3 has-text-centered">{{  title }}</h1>
+
+    {% if not status %}
+    <p class="mt-5 has-text-justified">
+        The latest update has brought some changes to the database and your current db is incompatible.
+        This will upgrade the database to work with the recent update.
+        The wizard will create a backup, so don't worry! You will find the
+        backup in the labertasche root directory under <span class="code">/backup/v{{ prev_version }}.zip</span>.
+        <br>
+        <span class="has-text-weight-bold has-text-danger" >Please do not reload this page during the process!</span>
+    </p>
+    <div class="field mt-5">
+        <div class="control" id="controls">
+            <button id="start-button" onclick="start_upgrade_to_v2();" class="button is-success">START</button>
+        </div>
+    </div>
+    <div class="content" id="update-messages"></div>
+    {% else %}
+        <p class="mt-5 has-text-justified">
+        This update has already run. Please return to the
+            <a href="{{ url_for('bp_dashboard.dashboard_project_list') }}">dashboard</a>.
+        </p>
+    {% endif %}
+</div>
+{% endblock %}
+{% block javascript_libs %}
+    <script src="/static/js/upgrade_to_v2.js"></script>
+{% endblock %}

templates/project-list.html

@@ -80,7 +80,7 @@
                         <div class="card-footer-item has-background-danger-dark">
                             <a class="has-text-weight-bold has-text-white is-uppercase"
                                data-tippy-content="Delete the project and all of its content"
-                               href="{{ url_for('bp_jsconnector.api_delete_project', project=each['name']) }}">DELETE</a>
+                               onclick="show_modal_with_project('modal-project-delete', '{{ each['name'] }}');">DELETE</a>
                         </div>
                         <div class="card-footer-item">
                             <a class="has-text-weight-bold has-text-black is-uppercase"
@@ -108,7 +108,7 @@
         <div class="modal-card">
             <header class="modal-card-head">
                 <p class="modal-card-title">New Project</p>
-                <button class="delete" aria-label="close"></button>
+                <button onclick="hide_modal('modal-new-project')" class="delete" aria-label="close"></button>
             </header>
             <section class="modal-card-body">
                 <label for="project-name" class="has-text-black">PROJECT NAME
@@ -133,7 +133,7 @@
         <div class="modal-card">
             <header class="modal-card-head has-background-danger">
                 <p class="modal-card-title has-text-white">ERROR</p>
-                <button class="delete" aria-label="close"></button>
+                <button onclick="hide_modal('modal-project-not-found')" class="delete" aria-label="close"></button>
             </header>
             <section class="modal-card-body has-text-black">
                 The specified project was not found! Did you delete it? Try refreshing the page.<br>
@@ -151,10 +151,94 @@
             </footer>
         </div>
     </div>
+    <div class="modal" id="modal-project-delete">
+        <div class="modal-background"></div>
+        <div class="modal-card">
+            <header class="modal-card-head has-background-warning">
+                <p class="modal-card-title has-text-black">WARNING!</p>
+                <button onclick="hide_modal('modal-project-delete')" class="delete" aria-label="close"></button>
+            </header>
+            <section class="modal-card-body has-text-black">
+                You are about to delete a project. All associated data will be unrecoverably lost!
+                Please perform a manual sql dump if you would like to retain that data.
+            </section>
+            <footer class="modal-card-foot">
+                <button id="modal-delete-ok" onclick="project_delete()" class="button is-danger">
+                    OK
+                </button>
+                <button id="modal-delete-cancel" onclick="hide_modal('modal-project-delete')" class="button is-success">
+                    CANCEL
+                </button>
+            </footer>
+        </div>
+    </div>
+    <div class="modal is-active" id="modal-project-edit">
+        <div class="modal-background"></div>
+        <div class="modal-card">
+            <header class="modal-card-head has-background-warning">
+                <p class="modal-card-title has-text-black">Edit Project</p>
+                <button onclick="hide_modal('modal-project-edit')" class="delete" aria-label="close"></button>
+            </header>
+            <section class="modal-card-body has-text-black bg-deepmatte">
+                <form>
+                    <div class="field">
+                        <div class="control">
+                            <input class="input" id="edit-project-name" name="edit-project-name" type="text">
+                        </div>
+                        <label class="label help has-text-white"  for="edit-project-name">
+                            Project Name
+                        </label>
+                    </div>
+                    <div class="field">
+                        <div class="control">
+                            <input class="input" id="edit-project-web-url" name="edit-project-web-url" type="text">
+                        </div>
+                        <label class="label help has-text-white" for="edit-project-web-url">
+                            URI of the comment system
+                        </label>
+                    </div>
+                    <div class="field">
+                        <div class="control">
+                            <input class="input" id="edit-project-blog-url" name="edit-project-blog-url" type="text">
+                        </div>
+                        <label class="label help has-text-white" for="edit-project-blog-url">
+                            URL of your Hugo site for this project
+                        </label>
+                    </div>
+                    <div class="field">
+                        <div class="control">
+                            <input class="input" id="edit-project-output" name="edit-project-output" type="text">
+                        </div>
+                        <label class="label help has-text-white" for="edit-project-output">Output Directory</label>
+                    </div>
+                    <div class="field">
+                        <div class="control">
+                            <label class="checkbox help has-text-white" for="edit-project-send-otp">
+                                <input id="edit-project-send-otp" class="checkbox" type="checkbox"
+                                       name="edit-project-send-otp" checked>
+                                Send OTP to publish?
+                            </label>
+                        </div>
+                    </div>
+                </form>
+            </section>
+            <footer class="modal-card-foot bg-deepmatte">
+                <button id="modal-delete-ok" onclick="project_delete()" class="button is-danger">
+                    OK
+                </button>
+                <button id="modal-delete-cancel" onclick="hide_modal('modal-project-edit')" class="button is-success">
+                    CANCEL
+                </button>
+            </footer>
+        </div>
+    </div>
 {% endblock %}
 {% block javascript %}
     tippy('[data-tippy-content]', {
         allowHTML: true,
         delay: 500
     });
+
+    weburl = document.getElementById('edit-project-web-url')
+    weburl.value = window.location.host;
 {% endblock %}