Database Upgrades

* Wrote a new module that allows database upgrades via the admin dashboard.
* Added a new table t_version that allows to check the version of the database.
* The upgrade path is for vanilla databases. It exports and backups data, then reintegrates it.
projects
Domeniko Gentner 3 years ago
parent 11b2fe5942
commit b831d5a8d5
  1. 1
      .gitignore
  2. 15
      labertasche.yaml
  3. 1
      labertasche/blueprints/__init__.py
  4. 14
      labertasche/blueprints/bp_upgrades/__init__.py
  5. 233
      labertasche/blueprints/bp_upgrades/db_v2.py
  6. 2
      labertasche/database/__init__.py
  7. 1
      labertasche/models/__init__.py
  8. 21
      labertasche/models/t_version.py
  9. 2
      labertasche/settings/__init__.py
  10. 48
      server.py
  11. 40
      templates/base.html
  12. 31
      templates/db-upgrades.html
  13. 90
      templates/project-list.html

1
.gitignore vendored

@ -7,3 +7,4 @@ db/*.db-wal
output
/output/
*.old
/backup/

@ -12,23 +12,10 @@ system:
database_uri: "sqlite:///db/labertasche.db" # Database URI. See documentation. Default is sqlite.
secret: "6Gxvb52bIJCm2vfDsmWKzShKp1omrzVG" # CHANGE ME! THIS IS IMPORTANT!
output: "./__implementation_example/data/" # Base path for the output json
debug: false # Leave this as is, this is for development.
debug: true # Leave this as is, this is for development.
send_otp_to_publish: true # Disables confirmation w/ OTP via mail
cookie_secure: false
projects:
- default:
web_url: "http://dev.localhost:1314/"
blog_url: "http://dev.localhost:1313/"
output: "./__implementation_example/data/"
send_otp_to_publish: true
- example.com:
web_url: "http://comments.example.com/"
blog_url: "http://blog.example.com/"
output: "./example/data/"
send_otp_to_publish: true
gravatar:
cache: true # Enable caching of gravatar images
static_dir: "./__implementation_example/static/images/gravatar/" # Where to store cached images, must exist!

@ -10,3 +10,4 @@ from .bp_comments import bp_comments
from .bp_login import bp_login
from .bp_dashboard import bp_dashboard
from .bp_jsconnector import bp_jsconnector
from .bp_upgrades import bp_dbupgrades

@ -0,0 +1,14 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# /**********************************************************************************
# * _author : Domeniko Gentner
# * _mail : code@tuxstash.de
# * _repo : https://git.tuxstash.de/gothseidank/labertasche
# * _license : This project is under MIT License
# *********************************************************************************/
from flask import Blueprint
# Blueprint
bp_dbupgrades = Blueprint("bp_dbupgrades", __name__, url_prefix='/upgrade')
from .db_v2 import upgrade_db_to_v2

@ -0,0 +1,233 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# /**********************************************************************************
# * _author : Domeniko Gentner
# * _mail : code@tuxstash.de
# * _repo : https://git.tuxstash.de/gothseidank/labertasche
# * _license : This project is under MIT License
# *********************************************************************************/
from . import bp_dbupgrades
from flask_cors import cross_origin
from flask_login import login_required
from flask import render_template, jsonify, make_response
from pathlib import Path
from labertasche.database import labertasche_db as db
from labertasche.models import TProjects, TComments, TLocation, TEmail, TVersion
from labertasche.helper import Settings
from json import dump, load
from shutil import copy, make_archive
from re import search
from secrets import compare_digest
from datetime import datetime
def get_backup_folder() -> Path:
path = Path('.').absolute() / "backup" / "v1"
return path
@cross_origin()
@bp_dbupgrades.route('/db_v2/')
@login_required
def upgrade_db_to_v2():
# TODO: Check if db has already been upgraded
status = False
try:
version = db.session.query(TVersion).first()
if version:
status = True
except Exception as e:
print(e.__class__)
pass
return render_template("db-upgrades.html", title="DB upgrade V1 to V2",
prev_version=1, new_version=2, status=status)
@cross_origin()
@bp_dbupgrades.route('/db_v2/backup/', methods=['GET'])
@login_required
def upgrade_db_to_v2_backup():
path = get_backup_folder()
# Create path for backup
try:
if not path.exists():
path.mkdir(mode=755, exist_ok=True, parents=True)
except OSError as e:
return make_response(jsonify(status='exception', msg=str(e)), 400)
return make_response(jsonify(status="ok"), 200)
@cross_origin()
@bp_dbupgrades.route('/db_v2/export/')
@login_required
def upgrade_db_to_v2_export():
path = get_backup_folder()
# make sure nothing is pending
db.session.commit()
# Export tables
t_locations = db.session.query(TLocation.id_location, TLocation.location).all()
t_emails = db.session.query(TEmail.id_email, TEmail.email, TEmail.is_allowed, TEmail.is_blocked).all()
t_comments = db.session.query(TComments.comments_id, TComments.location_id, TComments.email,
TComments.content, TComments.created_on, TComments.is_published,
TComments.is_spam, TComments.spam_score, TComments.replied_to,
TComments.confirmation, TComments.deletion, TComments.gravatar).all()
locations = []
for loc in t_locations:
locations.append({
"id_location": loc.id_location,
"location": loc.location
})
emails = []
for mail in t_emails:
emails.append({
"id_email": mail.id_email,
"email": mail.email,
"is_allowed": mail.is_allowed,
"is_blocked": mail.is_blocked
})
comments = []
for comment in t_comments:
comments.append({
"comments_id": comment.comments_id,
"location_id": comment.location_id,
"email": comment.email,
"content": comment.content,
"created_on": f"{comment.created_on.__str__()}",
"is_published": comment.is_published,
"is_spam": comment.is_spam,
"spam_score": comment.spam_score,
"replied_to": comment.replied_to,
"confirmation": comment.confirmation,
"deletion": comment.deletion,
"gravatar": comment.gravatar
})
# Output jsons
try:
p_export_location = path / "locations.json"
with p_export_location.open('w') as fp:
dump(locations, fp, indent=4, sort_keys=True)
p_export_mail = path / "emails.json"
with p_export_mail.open('w') as fp:
dump(emails, fp, indent=4, sort_keys=True)
p_export_comments = path / "comments.json"
with p_export_comments.open('w') as fp:
dump(comments, fp, indent=4, sort_keys=True)
except Exception as e:
return make_response(jsonify(status='exception-write-json', msg=str(e)), 400)
# Copy database
try:
settings = Settings()
db_uri = settings.system['database_uri']
if compare_digest(db_uri[0:6], "sqlite"):
m = search("([/]{3})(.*)", db_uri)
new_db = get_backup_folder() / "labertasche.db"
old_db = Path(m.group(2)).absolute()
copy(old_db, new_db)
except Exception as e:
return make_response(jsonify(status='exception-copy-db', msg=str(e)), 400)
make_archive(path, "zip", path)
return make_response(jsonify(status='ok'), 200)
@cross_origin()
@bp_dbupgrades.route('/db_v2/recreate/')
@login_required
def upgrade_db_to_v2_recreate():
try:
db.drop_all()
db.session.flush()
db.session.commit()
db.create_all()
except Exception as e:
return make_response(jsonify(status='exception', msg=str(e)), 400)
return make_response(jsonify(status='ok'), 200)
@cross_origin()
@bp_dbupgrades.route('/db_v2/import/')
@login_required
def upgrade_db_to_v2_import():
path = get_backup_folder()
settings = Settings()
try:
# load location
p_loc = (path / 'locations.json').absolute()
with p_loc.open('r') as fp:
locations = load(fp)
# load mails
m_loc = (path / 'emails.json').absolute()
with m_loc.open('r') as fp:
mails = load(fp)
# load comments
c_loc = (path / 'comments.json').absolute()
with c_loc.open('r') as fp:
comments = load(fp)
except FileNotFoundError as e:
return make_response(jsonify(status='exception-filenotfound', msg=str(e)), 400)
# Create project
default_project = {
"id_project": 1,
"name": "default",
"weburl": settings.system['web_url'],
"blogurl": settings.system['blog_url'],
"output": settings.system['output'],
"sendotp": settings.system['send_otp_to_publish'],
"gravatar_cache": settings.gravatar['cache'],
"gravatar_cache_dir": settings.gravatar['static_dir'],
"gravatar_size": settings.gravatar['size'],
"addon_smileys": settings.addons['smileys']
}
# Create db version, so we can track it in the future
version = {
"id_version": 1,
"version": 2
}
try:
# Add to db
db.session.add(TVersion(**version))
db.session.add(TProjects(**default_project))
# walk json and readd to database with project set to project 1
for each in mails:
each.update({'project_id': 1})
db.session.add(TEmail(**each))
for each in locations:
each.update({'project_id': 1})
db.session.add(TLocation(**each))
for each in comments:
each.update({'project_id': 1})
dt = datetime.fromisoformat(each['created_on'])
each.update({'created_on': dt})
db.session.add(TComments(**each))
# Commit
db.session.commit()
db.session.flush()
except Exception as e:
return make_response(jsonify(status='exception-database', msg=str(e)), 400)
return make_response(jsonify(status='ok'), 200)

@ -13,7 +13,7 @@ from sqlalchemy import MetaData
convention = {
"ix": 'ix_%(column_0_label)s',
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"ck": "ck_%(table_name)s_%(column_0_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s"
}

@ -10,3 +10,4 @@ from .t_comments import TComments
from .t_location import TLocation
from .t_emails import TEmail
from .t_projects import TProjects
from .t_version import TVersion

@ -0,0 +1,21 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# /**********************************************************************************
# * _author : Domeniko Gentner
# * _mail : code@tuxstash.de
# * _repo : https://git.tuxstash.de/gothseidank/labertasche
# * _license : This project is under MIT License
# *********************************************************************************/
from labertasche.database import labertasche_db as db
class TVersion(db.Model):
# table name
__tablename__ = "t_version"
__table_args__ = {'useexisting': True}
# primary key
id_version = db.Column(db.Integer, primary_key=True, autoincrement=True)
# data
version = db.Column(db.Integer)

@ -28,4 +28,4 @@ class Settings:
self.gravatar = conf['gravatar']
self.addons = conf['addons']
self.smileys = conf['smileys']
self.projects = conf['projects']

@ -9,17 +9,15 @@
import logging
from flask import Flask, redirect, url_for
from flask_cors import CORS
from sqlalchemy import event
from sqlalchemy import event, inspect
# noinspection PyProtectedMember
from sqlalchemy.engine import Engine
from labertasche.settings import Settings
from labertasche.database import labertasche_db
from labertasche.blueprints import bp_comments, bp_login, bp_dashboard, bp_jsconnector
from labertasche.models import TProjects
from labertasche.blueprints import bp_comments, bp_login, bp_dashboard, bp_jsconnector, bp_dbupgrades
from labertasche.helper import User
from flask_login import LoginManager
from flask_migrate import Migrate
from datetime import timedelta
# Load settings
settings = Settings()
@ -30,48 +28,48 @@ laberflask.config.update(dict(
SESSION_COOKIE_DOMAIN=settings.system['cookie_domain'],
SESSION_COOKIE_SECURE=settings.system['cookie_secure'],
REMEMBER_COOKIE_SECURE=settings.system['cookie_secure'],
REMEMBER_COOKIE_DURATION=timedelta(days=7),
REMEMBER_COOKIE_HTTPONLY=True,
REMEMBER_COOKIE_REFRESH_EACH_REQUEST=True,
DEBUG=settings.system['debug'],
SECRET_KEY=settings.system['secret'],
TEMPLATES_AUTO_RELOAD=True,
TEMPLATES_AUTO_RELOAD=settings.system['debug'],
SQLALCHEMY_DATABASE_URI=settings.system['database_uri'],
SQLALCHEMY_TRACK_MODIFICATIONS=False
))
# Flask migrate
migrate = Migrate(laberflask, labertasche_db, render_as_batch=True)
# Initialize ORM
labertasche_db.init_app(laberflask)
with laberflask.app_context():
labertasche_db.create_all()
project = labertasche_db.session.query(TProjects).filter(TProjects.id_project == 1).first()
if not project:
default_project = {
"id_project": 1,
"name": "default"
}
labertasche_db.session.add(TProjects(**default_project))
labertasche_db.session.commit()
# CORS
CORS(laberflask, resources={r"/comments": {"origins": settings.system['blog_url']}})
CORS(laberflask, resources={r"/comments": {"origins": settings.system['blog_url']},
r"/api": {"origins": settings.system['web_url']},
r"/dashboard": {"origins": settings.system['web_url']},
})
# Import blueprints
laberflask.register_blueprint(bp_comments)
laberflask.register_blueprint(bp_dashboard)
laberflask.register_blueprint(bp_login)
laberflask.register_blueprint(bp_jsconnector)
laberflask.register_blueprint(bp_dbupgrades)
# Disable Werkzeug's verbosity during development
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
# Set up login manager
loginmgr = LoginManager(laberflask)
loginmgr.login_view = 'bp_admin_login.login'
# Initialize ORM
labertasche_db.init_app(laberflask)
with laberflask.app_context():
table_names = inspect(labertasche_db.get_engine()).get_table_names()
is_empty = table_names == []
# Only create tables if the db is empty, so we can a controlled upgrade.
if is_empty:
labertasche_db.create_all()
# There is only one user
@loginmgr.user_loader
def user_loader(user_id):
if user_id != "0":
@ -79,11 +77,13 @@ def user_loader(user_id):
return User(user_id)
# User not authorized
@loginmgr.unauthorized_handler
def login_invalid():
return redirect(url_for('bp_login.show_login'))
# Enable write-ahead-log for sqlite databases
@event.listens_for(Engine, "connect")
def set_sqlite_pragma(dbapi_connection, connection_record):
if settings.system["database_uri"][0:6] == 'sqlite':

@ -4,15 +4,16 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, user-scalable=yes">
<meta name="description" content="labertasche comment system dashboard">
<link rel="preconnect" href="https://cdn.materialdesignicons.com">
<!-- Preload -->
<link rel="preload" href="/static/css/open-sans-v18-latin-regular.woff2" as="font" type="font/woff2" crossorigin="anonymous" media="all">
<link rel="preload" as="style" href="/static/css/labertasche.css">
<link rel="preload" href="https://cdn.materialdesignicons.com/5.4.55/css/materialdesignicons.min.css" as="style">
<link rel="preload" href="/static/css/materialdesignicons-webfont.woff2" as="font" type="font/woff2" crossorigin="anonymous" media="all">
<link rel="preload" href="/static/css/labertasche.css" as="style" crossorigin="anonymous" media="screen">
<link rel="preload" href="/static/css/materialdesignicons.min.css" as="style" crossorigin="anonymous" media="screen">
<link rel="preload" href="/static/css/Chart.min.css" as="style" media="screen">
<link rel="stylesheet" href="https://cdn.materialdesignicons.com/5.4.55/css/materialdesignicons.min.css">
<link rel="stylesheet" href="/static/css/labertasche.css" media="screen">
<link rel="stylesheet" href="/static/css/materialdesignicons.min.css" media="screen">
<link rel="stylesheet" href="/static/css/Chart.min.css" media="screen">
<title>labertasche Dashboard</title>
@ -49,21 +50,22 @@
{% block main %}
{% endblock %}
</section>
<script defer src="/static/js/dashboard.js"></script>
<script defer src="/static/js/Chart.bundle.min.js"></script>
<script defer src="https://unpkg.com/@popperjs/core@2"></script>
<script defer src="https://unpkg.com/tippy.js@6"></script>
<script defer>
document.addEventListener('DOMContentLoaded', () => {
// Comments
const urlParams = new URLSearchParams(window.location.search);
if (urlParams.get("error") === "404"){
show_modal('modal-project-not-found');
}
{% block javascript %}
<script defer src="/static/js/dashboard.js"></script>
<script defer src="/static/js/Chart.bundle.min.js"></script>
<script defer src="/static/js/popper.min.js"></script>
<script defer src="/static/js/tippy-bundle.umd.min.js"></script>
{% block javascript_libs %}
{% endblock %}
});
</script>
<script defer>
document.addEventListener('DOMContentLoaded', () => {
const urlParams = new URLSearchParams(window.location.search);
if (urlParams.get("error") === "404"){
show_modal('modal-project-not-found');
}
{% block javascript %}
{% endblock %}
});
</script>
</body>
</html>

@ -0,0 +1,31 @@
{% extends "base.html" %}
{% block main %}
<div style="min-height: 80vh;" class="container bg-deepmatte p-6 brdr-yayellow is-size-5">
<h1 class="is-size-3 has-text-centered">{{ title }}</h1>
{% if not status %}
<p class="mt-5 has-text-justified">
The latest update has brought some changes to the database and your current db is incompatible.
This will upgrade the database to work with the recent update.
The wizard will create a backup, so don't worry! You will find the
backup in the labertasche root directory under <span class="code">/backup/v{{ prev_version }}.zip</span>.
<br>
<span class="has-text-weight-bold has-text-danger" >Please do not reload this page during the process!</span>
</p>
<div class="field mt-5">
<div class="control" id="controls">
<button id="start-button" onclick="start_upgrade_to_v2();" class="button is-success">START</button>
</div>
</div>
<div class="content" id="update-messages"></div>
{% else %}
<p class="mt-5 has-text-justified">
This update has already run. Please return to the
<a href="{{ url_for('bp_dashboard.dashboard_project_list') }}">dashboard</a>.
</p>
{% endif %}
</div>
{% endblock %}
{% block javascript_libs %}
<script src="/static/js/upgrade_to_v2.js"></script>
{% endblock %}

@ -80,7 +80,7 @@
<div class="card-footer-item has-background-danger-dark">
<a class="has-text-weight-bold has-text-white is-uppercase"
data-tippy-content="Delete the project and all of its content"
href="{{ url_for('bp_jsconnector.api_delete_project', project=each['name']) }}">DELETE</a>
onclick="show_modal_with_project('modal-project-delete', '{{ each['name'] }}');">DELETE</a>
</div>
<div class="card-footer-item">
<a class="has-text-weight-bold has-text-black is-uppercase"
@ -108,7 +108,7 @@
<div class="modal-card">
<header class="modal-card-head">
<p class="modal-card-title">New Project</p>
<button class="delete" aria-label="close"></button>
<button onclick="hide_modal('modal-new-project')" class="delete" aria-label="close"></button>
</header>
<section class="modal-card-body">
<label for="project-name" class="has-text-black">PROJECT NAME
@ -133,7 +133,7 @@
<div class="modal-card">
<header class="modal-card-head has-background-danger">
<p class="modal-card-title has-text-white">ERROR</p>
<button class="delete" aria-label="close"></button>
<button onclick="hide_modal('modal-project-not-found')" class="delete" aria-label="close"></button>
</header>
<section class="modal-card-body has-text-black">
The specified project was not found! Did you delete it? Try refreshing the page.<br>
@ -151,10 +151,94 @@
</footer>
</div>
</div>
<div class="modal" id="modal-project-delete">
<div class="modal-background"></div>
<div class="modal-card">
<header class="modal-card-head has-background-warning">
<p class="modal-card-title has-text-black">WARNING!</p>
<button onclick="hide_modal('modal-project-delete')" class="delete" aria-label="close"></button>
</header>
<section class="modal-card-body has-text-black">
You are about to delete a project. All associated data will be unrecoverably lost!
Please perform a manual sql dump if you would like to retain that data.
</section>
<footer class="modal-card-foot">
<button id="modal-delete-ok" onclick="project_delete()" class="button is-danger">
OK
</button>
<button id="modal-delete-cancel" onclick="hide_modal('modal-project-delete')" class="button is-success">
CANCEL
</button>
</footer>
</div>
</div>
<div class="modal is-active" id="modal-project-edit">
<div class="modal-background"></div>
<div class="modal-card">
<header class="modal-card-head has-background-warning">
<p class="modal-card-title has-text-black">Edit Project</p>
<button onclick="hide_modal('modal-project-edit')" class="delete" aria-label="close"></button>
</header>
<section class="modal-card-body has-text-black bg-deepmatte">
<form>
<div class="field">
<div class="control">
<input class="input" id="edit-project-name" name="edit-project-name" type="text">
</div>
<label class="label help has-text-white" for="edit-project-name">
Project Name
</label>
</div>
<div class="field">
<div class="control">
<input class="input" id="edit-project-web-url" name="edit-project-web-url" type="text">
</div>
<label class="label help has-text-white" for="edit-project-web-url">
URI of the comment system
</label>
</div>
<div class="field">
<div class="control">
<input class="input" id="edit-project-blog-url" name="edit-project-blog-url" type="text">
</div>
<label class="label help has-text-white" for="edit-project-blog-url">
URL of your Hugo site for this project
</label>
</div>
<div class="field">
<div class="control">
<input class="input" id="edit-project-output" name="edit-project-output" type="text">
</div>
<label class="label help has-text-white" for="edit-project-output">Output Directory</label>
</div>
<div class="field">
<div class="control">
<label class="checkbox help has-text-white" for="edit-project-send-otp">
<input id="edit-project-send-otp" class="checkbox" type="checkbox"
name="edit-project-send-otp" checked>
Send OTP to publish?
</label>
</div>
</div>
</form>
</section>
<footer class="modal-card-foot bg-deepmatte">
<button id="modal-delete-ok" onclick="project_delete()" class="button is-danger">
OK
</button>
<button id="modal-delete-cancel" onclick="hide_modal('modal-project-edit')" class="button is-success">
CANCEL
</button>
</footer>
</div>
</div>
{% endblock %}
{% block javascript %}
tippy('[data-tippy-content]', {
allowHTML: true,
delay: 500
});
weburl = document.getElementById('edit-project-web-url')
weburl.value = window.location.host;
{% endblock %}

Loading…
Cancel
Save