From bf3278a6b00d1ae94d0c5355ad68108fab0ec45a Mon Sep 17 00:00:00 2001 From: Chris Clark Date: Fri, 7 Jun 2024 17:03:45 -0400 Subject: [PATCH] Light branding, and bugixes to user uploads (#626) * branding and user upload fixes * 5.0b1 release --- HISTORY.rst | 3 +- MANIFEST.in | 2 +- explorer/__init__.py | 6 +- explorer/ee/db_connections/models.py | 3 +- explorer/ee/db_connections/utils.py | 22 +- explorer/ee/db_connections/views.py | 2 +- explorer/schema.py | 7 +- explorer/src/images/logo.svg | 14 + explorer/src/js/uploads.js | 53 ++-- explorer/src/scss/explorer.scss | 8 + explorer/src/scss/styles.scss | 4 +- explorer/src/scss/variables.scss | 20 +- .../connections/database_connection_form.html | 8 +- explorer/templates/explorer/base.html | 10 +- .../templates/explorer/query_favorites.html | 2 +- explorer/templatetags/vite.py | 19 +- ...n_utils.py => test_db_connection_utils.py} | 0 explorer/tests/test_models.py | 20 ++ explorer/tests/test_views.py | 3 +- explorer/views/query.py | 24 +- package-lock.json | 248 ++++++++++++++++-- package.json | 17 +- test_project/settings.py | 2 +- vite.config.js => vite.config.mjs | 16 +- 24 files changed, 431 insertions(+), 82 deletions(-) create mode 100644 explorer/src/images/logo.svg rename explorer/tests/{test_user_connection_utils.py => test_db_connection_utils.py} (100%) rename vite.config.js => vite.config.mjs (78%) diff --git a/HISTORY.rst b/HISTORY.rst index e3e4e945..227dac06 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,7 +5,7 @@ Change Log This document records all notable changes to `django-sql-explorer `_. This project adheres to `Semantic Versioning `_. -`4.4.0b1`_ (2024-06-02) +`5.0.0b1`_ (2024-06-07) =========================== * Manage DB connections via the UI (and/or Django Admin) @@ -18,6 +18,7 @@ This project adheres to `Semantic Versioning `_. * Query List home page is sortable. * Select all / deselect all with AI assistant * Assistant tests run reliably in CI/CD +* Introduced some branding and styling improvements `4.3.0`_ (2024-05-27) diff --git a/MANIFEST.in b/MANIFEST.in index c8d6d1de..1f55348f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ recursive-include explorer * recursive-exclude * *.pyc __pycache__ .DS_Store recursive-include requirements * include package.json -include vite.config.js +include vite.config.mjs include README.rst diff --git a/explorer/__init__.py b/explorer/__init__.py index 5e4bdc17..57bb0fcf 100644 --- a/explorer/__init__.py +++ b/explorer/__init__.py @@ -1,9 +1,9 @@ __version_info__ = { - "major": 4, - "minor": 4, + "major": 5, + "minor": 0, "patch": 0, "releaselevel": "beta", - "serial": 3 + "serial": 1 } diff --git a/explorer/ee/db_connections/models.py b/explorer/ee/db_connections/models.py index 627a436e..c8c8bc22 100644 --- a/explorer/ee/db_connections/models.py +++ b/explorer/ee/db_connections/models.py @@ -5,6 +5,7 @@ from django.db import models from django.db.models.signals import pre_save from django.dispatch import receiver +from explorer.ee.db_connections.utils import user_dbs_local_dir from django_cryptography.fields import encrypt @@ -41,7 +42,7 @@ def is_upload(self): @property def local_name(self): if self.is_upload: - return os.path.normpath(os.path.join(os.getcwd(), f"user_dbs/{self.name}")) + return os.path.join(user_dbs_local_dir(), self.name) @classmethod def from_django_connection(cls, connection_alias): diff --git a/explorer/ee/db_connections/utils.py b/explorer/ee/db_connections/utils.py index 7eae7e30..c43a3d98 100644 --- a/explorer/ee/db_connections/utils.py +++ b/explorer/ee/db_connections/utils.py @@ -10,17 +10,26 @@ import io -# TODO deal with uploading the same file / conflicting file again +# Uploading the same filename twice (from the same user) will overwrite the 'old' DB on S3 def upload_sqlite(db_bytes, path): from explorer.utils import get_s3_bucket bucket = get_s3_bucket() bucket.put_object(Key=path, Body=db_bytes, ServerSideEncryption="AES256") -def create_connection_for_uploaded_sqlite(filename, s3_path): +# Aliases have the user_id appended to them so that if two users upload files with the same name +# they don't step on one another. Without this, the *files* would get uploaded separately (because +# the DBs go into user-specific folders on s3), but the *aliases* would be the same. So one user +# could (innocently) upload a file with the same name, and any existing queries would be suddenly pointing +# to this new database connection. Oops! +# TODO: In the future, queries should probably be FK'ed to the ID of the connection, rather than simply +# storing the alias of the connection as a string. +def create_connection_for_uploaded_sqlite(filename, user_id, s3_path): from explorer.models import DatabaseConnection + base, ext = os.path.splitext(filename) + filename = f"{base}_{user_id}{ext}" return DatabaseConnection.objects.create( - alias=filename, + alias=f"{filename}", engine=DatabaseConnection.SQLITE, name=filename, host=s3_path @@ -41,6 +50,13 @@ def get_sqlite_for_connection(explorer_connection): return explorer_connection +def user_dbs_local_dir(): + d = os.path.normpath(os.path.join(os.getcwd(), "user_dbs")) + if not os.path.exists(d): + os.makedirs(d) + return d + + def create_django_style_connection(explorer_connection): if explorer_connection.is_upload: diff --git a/explorer/ee/db_connections/views.py b/explorer/ee/db_connections/views.py index 40fc3512..262e8aa9 100644 --- a/explorer/ee/db_connections/views.py +++ b/explorer/ee/db_connections/views.py @@ -49,7 +49,7 @@ def post(self, request): except Exception as e: # noqa return JsonResponse({"error": "Error while uploading file."}, status=400) - create_connection_for_uploaded_sqlite(f_name, s3_path) + create_connection_for_uploaded_sqlite(f_name, request.user.id, s3_path) return JsonResponse({"success": True}) else: return JsonResponse({"error": "No file provided"}, status=400) diff --git a/explorer/schema.py b/explorer/schema.py index b73c6147..db2e7a6c 100644 --- a/explorer/schema.py +++ b/explorer/schema.py @@ -5,7 +5,7 @@ EXPLORER_SCHEMA_INCLUDE_TABLE_PREFIXES, EXPLORER_SCHEMA_INCLUDE_VIEWS, ) from explorer.tasks import build_schema_cache_async -from explorer.utils import get_valid_connection +from explorer.utils import get_valid_connection, InvalidExplorerConnectionException # These wrappers make it easy to mock and test @@ -53,7 +53,10 @@ def schema_json_info(connection_alias): ret = cache.get(key) if ret: return ret - si = schema_info(connection_alias) or [] + try: + si = schema_info(connection_alias) or [] + except InvalidExplorerConnectionException: + return [] json_schema = transform_to_json_schema(si) cache.set(key, json_schema) return json_schema diff --git a/explorer/src/images/logo.svg b/explorer/src/images/logo.svg new file mode 100644 index 00000000..60407416 --- /dev/null +++ b/explorer/src/images/logo.svg @@ -0,0 +1,14 @@ + + + + + + + + + \ No newline at end of file diff --git a/explorer/src/js/uploads.js b/explorer/src/js/uploads.js index c1b9a16e..2b005313 100644 --- a/explorer/src/js/uploads.js +++ b/explorer/src/js/uploads.js @@ -3,6 +3,8 @@ import { getCsrfToken } from "./csrf"; export function setupUploads() { var dropArea = document.getElementById('drop-area'); var fileElem = document.getElementById('fileElem'); + var progressBar = document.getElementById('progress-bar'); + var uploadStatus = document.getElementById('upload-status'); if (dropArea) { dropArea.onclick = function() { @@ -10,17 +12,17 @@ export function setupUploads() { }; dropArea.addEventListener('dragover', function(e) { - e.preventDefault(); // Prevent default behavior (Prevent file from being opened) - dropArea.classList.add('bg-info'); // Optional: add a style when dragging over + e.preventDefault(); + dropArea.classList.add('bg-info'); }); dropArea.addEventListener('dragleave', function(e) { - dropArea.classList.remove('bg-info'); // Optional: remove style when not dragging over + dropArea.classList.remove('bg-info'); }); dropArea.addEventListener('drop', function(e) { e.preventDefault(); - dropArea.classList.remove('bg-info'); // Optional: remove style after dropping + dropArea.classList.remove('bg-info'); let files = e.dataTransfer.files; if (files.length) { @@ -35,8 +37,6 @@ export function setupUploads() { }; } - - function handleFiles(file) { uploadFile(file); } @@ -45,17 +45,35 @@ export function setupUploads() { let formData = new FormData(); formData.append('file', file); - fetch("../upload/", { - method: 'POST', - headers: { - 'X-CSRFToken': getCsrfToken() - }, - body: formData - }).then(response => response.json()) - .then(() => { - window.location.reload(); - }) - .catch(error => console.error('Error:', error)); + let xhr = new XMLHttpRequest(); + xhr.open('POST', '../upload/', true); + xhr.setRequestHeader('X-CSRFToken', getCsrfToken()); + + xhr.upload.onprogress = function(event) { + if (event.lengthComputable) { + let percentComplete = (event.loaded / event.total) * 100; + progressBar.style.width = percentComplete + '%'; + progressBar.setAttribute('aria-valuenow', percentComplete); + progressBar.innerHTML = percentComplete.toFixed(0) + '%'; + if (percentComplete > 99) { + uploadStatus.innerHTML = "Upload complete. Parsing and saving to S3..."; + } + } + }; + + xhr.onload = function() { + if (xhr.status === 200) { + window.location.href = "../"; + } else { + console.error('Error:', xhr.statusText); + } + }; + + xhr.onerror = function() { + console.error('Error:', xhr.statusText); + }; + + xhr.send(formData); } document.getElementById("test-connection-btn").addEventListener("click", function() { @@ -79,5 +97,4 @@ export function setupUploads() { }) .catch(error => console.error("Error:", error)); }); - } diff --git a/explorer/src/scss/explorer.scss b/explorer/src/scss/explorer.scss index f8390283..7aedda51 100644 --- a/explorer/src/scss/explorer.scss +++ b/explorer/src/scss/explorer.scss @@ -1,3 +1,7 @@ +a { + text-decoration: none; +} + .cm-editor { outline: none !important; } @@ -143,3 +147,7 @@ nav.navbar { border-radius: 4px; box-shadow: 0 2px 5px rgba(0,0,0,0.2); } + +.logo-image { + height: 2rem; +} diff --git a/explorer/src/scss/styles.scss b/explorer/src/scss/styles.scss index d19f6c8b..cece367d 100644 --- a/explorer/src/scss/styles.scss +++ b/explorer/src/scss/styles.scss @@ -1,9 +1,11 @@ +@import "variables"; + @import "~bootstrap/scss/bootstrap"; $bootstrap-icons-font-dir: "../../../node_modules/bootstrap-icons/font/fonts"; @import "~bootstrap-icons/font/bootstrap-icons.css"; -@import "variables"; + @import "explorer"; @import "assistant"; diff --git a/explorer/src/scss/variables.scss b/explorer/src/scss/variables.scss index 708f726a..90301d2a 100644 --- a/explorer/src/scss/variables.scss +++ b/explorer/src/scss/variables.scss @@ -1,3 +1,19 @@ -:root { - --bs-dark-rgb: 44, 62, 80; +@import "../../../node_modules/bootstrap/scss/functions"; + +$orange: rgb(255, 80, 1); +$blue: rgb(3, 68, 220); +$green: rgb(127, 176, 105); +$primary: $blue; +$dark: rgb(1, 32, 63); +$secondary: $orange; +$warning: $orange; +$danger: $orange; +$success: $green; +$info: rgb(106, 141, 146); +$code-color: $info; +$font-size-base: .8rem; + +.btn-secondary, .btn-info { + --bs-btn-color: white !important; } + diff --git a/explorer/templates/connections/database_connection_form.html b/explorer/templates/connections/database_connection_form.html index cce67638..f5ef5260 100644 --- a/explorer/templates/connections/database_connection_form.html +++ b/explorer/templates/connections/database_connection_form.html @@ -14,8 +14,12 @@

{% if object %}Edit{% else %}Create New{% endif %} Connection

...or upload a SQLite DB or CSV File

CSV files will get parsed and typed automatically. SQLite databases must not be password protected.

-

Drag and drop .csv or SQLite .db or click to upload.

- +

Drag and drop .csv or SQLite .db/.sqlite or click to upload.

+ +
+
0%
+
+

{% endif %} diff --git a/explorer/templates/explorer/base.html b/explorer/templates/explorer/base.html index fbc0d7db..4f5b2201 100644 --- a/explorer/templates/explorer/base.html +++ b/explorer/templates/explorer/base.html @@ -7,6 +7,7 @@ {% trans "SQL Explorer" %}{% if query %} - {{ query.title }}{% elif title %} - {{ title }}{% endif %} + {% block style %} {% vite_asset 'scss/styles.scss' %} @@ -42,8 +43,11 @@

This is easy to fix, I promise!

{% block sql_explorer_content_takeover %}