Skip to content

Commit

Permalink
dont rely on locale in case it is not available (#629)
Browse files Browse the repository at this point in the history
5.0b2
  • Loading branch information
chrisclark authored Jun 17, 2024
1 parent 503f362 commit a3cbe82
Show file tree
Hide file tree
Showing 16 changed files with 123 additions and 60 deletions.
10 changes: 6 additions & 4 deletions HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,19 @@ Change Log
This document records all notable changes to `django-sql-explorer <https://github.com/chrisclark/django-sql-explorer>`_.
This project adheres to `Semantic Versioning <https://semver.org/>`_.

`5.0.0b1`_ (2024-06-07)
`5.0.b2`_ (2024-06-17)
===========================

* Manage DB connections via the UI (and/or Django Admin)
* Manage DB connections via the UI (and/or Django Admin). Set EXPLORER_DB_CONNECTIONS_ENABLED
to True in settings to enable user-facing connection management.
* Upload CSV or SQLite DBs directly, to create additional connections.
This functionality has additional dependencies which can be installed with
the 'uploads' extra (e.g. pip install django-sql-explorer[uploads])
the 'uploads' extra (e.g. pip install django-sql-explorer[uploads]). Then set EXPLORER_USER_UPLOADS_ENABLED
to True, and make sure S3_BUCKET is also set up.
* The above functionality is managed by a new license, restricting the
ability of 3rd parties resell SQL Explorer (commercial usage is absolutely
still permitted).
* Query List home page is sortable.
* Query List home page is sortable
* Select all / deselect all with AI assistant
* Assistant tests run reliably in CI/CD
* Introduced some branding and styling improvements
Expand Down
2 changes: 1 addition & 1 deletion explorer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"minor": 0,
"patch": 0,
"releaselevel": "beta",
"serial": 1
"serial": 2
}


Expand Down
16 changes: 14 additions & 2 deletions explorer/app_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,22 @@
"max_tokens": 128000})

EXPLORER_DB_CONNECTIONS_ENABLED = getattr(settings, "EXPLORER_DB_CONNECTIONS_ENABLED", False)
EXPLORER_USER_UPLOADS_ENABLED = getattr(settings, "EXPLORER_USER_UPLOADS_ENABLED", False)
EXPLORER_PRUNE_LOCAL_UPLOAD_COPY_DAYS_INACTIVITY = getattr(settings,
"EXPLORER_PRUNE_LOCAL_UPLOAD_COPY_DAYS_INACTIVITY", 7)
# 500mb default max
EXPLORER_MAX_UPLOAD_SIZE = getattr(settings, "EXPLORER_MAX_UPLOAD_SIZE", 500 * 1024 * 1024)

def has_assistant(): return EXPLORER_AI_API_KEY is not None
def db_connections_enabled(): return EXPLORER_DB_CONNECTIONS_ENABLED

def has_assistant():
return EXPLORER_AI_API_KEY is not None


def db_connections_enabled():
return EXPLORER_DB_CONNECTIONS_ENABLED


def user_uploads_enabled():
return (EXPLORER_USER_UPLOADS_ENABLED and
EXPLORER_DB_CONNECTIONS_ENABLED and
S3_BUCKET is not None)
22 changes: 17 additions & 5 deletions explorer/ee/db_connections/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from django.db.utils import load_backend
import os

import locale
from dateutil import parser

import pandas as pd
Expand Down Expand Up @@ -112,12 +111,25 @@ def pandas_to_sqlite(df, local_path="local_database.db"):
SHORTEST_PLAUSIBLE_DATE_STRING = 5


def atof_custom(value):
# Remove any thousands separators and convert the decimal point
if "," in value and "." in value:
if value.index(",") < value.index("."):
# 0,000.00 format
value = value.replace(",", "")
else:
# 0.000,00 format
value = value.replace(".", "").replace(",", ".")
elif "," in value:
# No decimal point, only thousands separator
value = value.replace(",", "")
return float(value)

def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True): # noqa
try:

csv_file = io.BytesIO(csv_bytes)
df = pd.read_csv(csv_file, sep=delimiter, header=0 if has_headers else None)
locale.setlocale(locale.LC_NUMERIC, "en_US.UTF-8")

for column in df.columns:
values = df[column].dropna().unique()
Expand All @@ -130,7 +142,7 @@ def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True): # noqa

for value in values:
try:
float_val = locale.atof(str(value))
float_val = atof_custom(str(value))
if float_val == int(float_val):
continue # This is effectively an integer
else:
Expand Down Expand Up @@ -163,12 +175,12 @@ def csv_to_typed_df(csv_bytes, delimiter=",", has_headers=True): # noqa
if is_date:
df[column] = pd.to_datetime(df[column], errors="coerce", utc=True)
elif is_integer:
df[column] = df[column].apply(lambda x: int(locale.atof(str(x))) if pd.notna(x) else x)
df[column] = df[column].apply(lambda x: int(atof_custom(str(x))) if pd.notna(x) else x)
# If there are NaN / blank values, the column will be converted to float
# Convert it back to integer
df[column] = df[column].astype("Int64")
elif is_float:
df[column] = df[column].apply(lambda x: locale.atof(str(x)) if pd.notna(x) else x)
df[column] = df[column].apply(lambda x: atof_custom(str(x)) if pd.notna(x) else x)
else:
inferred_type = pd.api.types.infer_dtype(values)
if inferred_type == "integer":
Expand Down
63 changes: 63 additions & 0 deletions explorer/src/images/logo-main.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added explorer/src/images/logo.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
14 changes: 0 additions & 14 deletions explorer/src/images/logo.svg

This file was deleted.

29 changes: 8 additions & 21 deletions explorer/src/scss/explorer.scss
Original file line number Diff line number Diff line change
Expand Up @@ -108,27 +108,6 @@ div.sort {
display: none;
}

nav.navbar {
.nav-link {
color: white;
}

nav.nav-link:hover, .nav-link:focus {
color: var(--bs-primary-bg-subtle);
}

.nav-pills .nav-link.active, .nav-pills .show > .nav-link {
color: var(--bs-nav-pills-link-active-color);
border: 1px solid var(--bs-secondary-bg);
background: none;
}

.nav-link:hover, .navbar-brand:hover {
background: none;
color: var(--bs-primary-bg-subtle) !important;
}
}

.dropdown-toggle::after {
margin-left: 0 !important;
}
Expand All @@ -151,3 +130,11 @@ nav.navbar {
.logo-image {
height: 2rem;
}

.stats-wrapper {
td {
padding-top: 0;
padding-bottom: 0;
border: none;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ <h2>{% if object %}Edit{% else %}Create New{% endif %} Connection</h2>
<a href="{% url 'explorer_connections' %}" class="btn btn-secondary">Cancel</a>
<button type="button" class="btn btn-info" id="test-connection-btn">Test Connection</button>
</form>
{% if not object and s3_enabled %}
{% if not object and user_uploads_enabled %}
<div class="pt-5">
<h4>...or upload a SQLite DB or CSV File</h4>
<p>CSV files will get parsed and typed automatically. SQLite databases must <i>not</i> be password protected.</p>
Expand Down
2 changes: 1 addition & 1 deletion explorer/templates/explorer/assistant.html
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
data-bs-toggle="tooltip" data-bs-placement="top"
data-bs-title="SQL Assistant builds a prompt with your query, your request, relevant schema (tables referenced in your query) and sample data from those tables. You can optionally include other tables (schema + data sample) that you'd like SQL Assistant to know about.">(?)</a>
<label for="search_assistant_tables" class="d-none" id="search_assistant_tables_label">Search tables</label>

<div id="table-list" class="list"></div>
</div>
</div>
Expand Down
8 changes: 4 additions & 4 deletions explorer/templates/explorer/base.html
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{% trans "SQL Explorer" %}{% if query %} - {{ query.title }}{% elif title %} - {{ title }}{% endif %}</title>
<link rel="icon" type="image/svg+xml" href="{% vite_asset 'images/logo.svg' %}">
<link rel="icon" type="image/svg+xml" href="{% vite_asset 'images/logo.png' %}">

{% block style %}
{% vite_asset 'scss/styles.scss' %}
Expand Down Expand Up @@ -41,11 +41,11 @@ <h2>This is easy to fix, I promise!</h2>
{% endif %}

{% block sql_explorer_content_takeover %}
<nav class="navbar navbar-expand-lg navbar-dark bg-dark mb-3">
<nav class="navbar navbar-expand-lg navbar-dark mb-3">
<div class="container bd-gutter flex-wrap flex-lg-nowrap">
<a class="navbar-brand{% if view_name == 'explorer_index' %} btn text-white{% endif %} d-flex align-items-center"
<a class="navbar-brand d-flex align-items-center"
href="{% url 'explorer_index' %}">
<img src="{% vite_asset 'images/logo.svg' %}" alt="Logo" class="me-2 logo-image" >
<img src="{% vite_asset 'images/logo-main.svg' %}" alt="Logo" class="me-2 logo-image" >
{% trans "SQL Explorer" %}
</a>
<ul class="nav nav-pills">
Expand Down
5 changes: 2 additions & 3 deletions explorer/templates/explorer/preview_pane.html
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
<th class="preview-header counter" style="display: none;"></th>
{% for h in headers %}
<th class="preview-header"><span><i class="sort bi-chevron-expand pe-1" data-sort="{{ forloop.counter0 }}"
data-dir="asc"></i>{{ h }}</span></th>
data-dir="asc"></i>{{ h }}{% if h.summary %}<i class="stats-expand bi-calculator text-info ps-1"></i>{% endif %}</span></th>
{% endfor %}
</tr>
{% if has_stats %}
Expand All @@ -68,11 +68,10 @@
{% for h in headers %}
<th>
{% if h.summary %}
<i class="stats-expand bi-calculator"></i>
<table class="stats-wrapper table table-sm fw-normal small" style="display: none;">
{% for label, value in h.summary.stats.items %}
<tr>
<td>{{ label }}</td>
<td class="text-info">{{ label }}</td>
<td>{{ value }}</td>
</tr>
{% endfor %}
Expand Down
4 changes: 2 additions & 2 deletions explorer/templates/explorer/query_list.html
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ <h3>
Recently Run by You
{% endblocktrans %}
</h3>
<table class="table table-striped">
<table class="table table-striped table-borderless">
<thead>
<tr>
<th>{% trans "Query" %}</th>
Expand Down Expand Up @@ -46,7 +46,7 @@ <h3>{% trans "All Queries" %}</h3>
<input class="search" placeholder="{% trans 'Search' %}" style="">
</div>
</div>
<table class="table table-striped">
<table class="table table-striped table-borderless">
<thead>
<tr>
<th><i class="sort bi-chevron-expand pe-1" data-sort="sort-name"
Expand Down
1 change: 1 addition & 0 deletions explorer/tests/csvs/floats.csv
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ Floats,Values
,4
"69.42719764194946",4
"2,000.0128",5
"2.000,0128",5
2 changes: 1 addition & 1 deletion explorer/views/mixins.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def gen_ctx(self):
),
"assistant_enabled": app_settings.has_assistant(),
"db_connections_enabled": app_settings.db_connections_enabled(),
"s3_enabled": app_settings.S3_BUCKET is not None,
"user_uploads_enabled": app_settings.user_uploads_enabled(),
"csrf_cookie_name": settings.CSRF_COOKIE_NAME,
"csrf_token_in_dom": settings.CSRF_COOKIE_HTTPONLY or settings.CSRF_USE_SESSIONS,
"view_name": self.request.resolver_match.view_name,
Expand Down
3 changes: 2 additions & 1 deletion test_project/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
SECRET_KEY = "shhh"
DEBUG = True
STATIC_URL = "/static/"
VITE_DEV_MODE = False
VITE_DEV_MODE = True

ALLOWED_HOSTS = ["0.0.0.0", "localhost", "127.0.0.1"]

Expand Down Expand Up @@ -113,3 +113,4 @@
EXPLORER_AI_API_KEY = os.environ.get("AI_API_KEY")
EXPLORER_ASSISTANT_BASE_URL = os.environ.get("AI_BASE_URL")
EXPLORER_DB_CONNECTIONS_ENABLED = True
EXPLORER_USER_UPLOADS_ENABLED = True

0 comments on commit a3cbe82

Please sign in to comment.