mirror of
https://github.com/frappe/frappe.git
synced 2024-06-03 02:10:48 +00:00
feat: Add identified index from UI
This commit is contained in:
Ankush Menat
parent
4e251e9b0b
commit
b169f8780a
|
|
@ -587,103 +587,6 @@ def add_db_index(context, doctype, column):
|
|||
raise SiteNotSpecifiedError
|
||||
|
||||
|
||||
@click.command("describe-database-table")
|
||||
@click.option("--doctype", help="DocType to describe")
|
||||
@click.option(
|
||||
"--column",
|
||||
multiple=True,
|
||||
help="Explicitly fetch accurate cardinality from table data. This can be quite slow on large tables.",
|
||||
)
|
||||
@pass_context
|
||||
def describe_database_table(context, doctype, column):
|
||||
"""Describes various statistics about the table.
|
||||
|
||||
This is useful to build integration like
|
||||
This includes:
|
||||
1. Schema
|
||||
2. Indexes
|
||||
3. stats - total count of records
|
||||
4. if column is specified then extra stats are generated for column:
|
||||
Distinct values count in column
|
||||
"""
|
||||
import json
|
||||
|
||||
for site in context.sites:
|
||||
frappe.init(site=site)
|
||||
frappe.connect()
|
||||
try:
|
||||
data = _extract_table_stats(doctype, column)
|
||||
# NOTE: Do not print anything else in this to avoid clobbering the output.
|
||||
print(json.dumps(data, indent=2))
|
||||
finally:
|
||||
frappe.destroy()
|
||||
|
||||
if not context.sites:
|
||||
raise SiteNotSpecifiedError
|
||||
|
||||
|
||||
def _extract_table_stats(doctype: str, columns: list[str]) -> dict:
|
||||
from frappe.utils import cint, cstr, get_table_name
|
||||
|
||||
def sql_bool(val):
|
||||
return cstr(val).lower() in ("yes", "1", "true")
|
||||
|
||||
table = get_table_name(doctype, wrap_in_backticks=True)
|
||||
|
||||
schema = []
|
||||
for field in frappe.db.sql(f"describe {table}", as_dict=True):
|
||||
schema.append(
|
||||
{
|
||||
"column": field["Field"],
|
||||
"type": field["Type"],
|
||||
"is_nullable": sql_bool(field["Null"]),
|
||||
"default": field["Default"],
|
||||
}
|
||||
)
|
||||
|
||||
def update_cardinality(column, value):
|
||||
for col in schema:
|
||||
if col["column"] == column:
|
||||
col["cardinality"] = value
|
||||
break
|
||||
|
||||
indexes = []
|
||||
for idx in frappe.db.sql(f"show index from {table}", as_dict=True):
|
||||
indexes.append(
|
||||
{
|
||||
"unique": not sql_bool(idx["Non_unique"]),
|
||||
"cardinality": idx["Cardinality"],
|
||||
"name": idx["Key_name"],
|
||||
"sequence": idx["Seq_in_index"],
|
||||
"nullable": sql_bool(idx["Null"]),
|
||||
"column": idx["Column_name"],
|
||||
"type": idx["Index_type"],
|
||||
}
|
||||
)
|
||||
if idx["Seq_in_index"] == 1:
|
||||
update_cardinality(idx["Column_name"], idx["Cardinality"])
|
||||
|
||||
total_rows = cint(
|
||||
frappe.db.sql(
|
||||
f"""select table_rows
|
||||
from information_schema.tables
|
||||
where table_name = 'tab{doctype}'"""
|
||||
)[0][0]
|
||||
)
|
||||
|
||||
# fetch accurate cardinality for columns by query. WARN: This can take a lot of time.
|
||||
for column in columns:
|
||||
cardinality = frappe.db.sql(f"select count(distinct {column}) from {table}")[0][0]
|
||||
update_cardinality(column, cardinality)
|
||||
|
||||
return {
|
||||
"table_name": table.strip("`"),
|
||||
"total_rows": total_rows,
|
||||
"schema": schema,
|
||||
"indexes": indexes,
|
||||
}
|
||||
|
||||
|
||||
@click.command("add-system-manager")
|
||||
@click.argument("email")
|
||||
@click.option("--first-name")
|
||||
|
|
@ -1602,7 +1505,6 @@ commands = [
|
|||
add_system_manager,
|
||||
add_user_for_sites,
|
||||
add_db_index,
|
||||
describe_database_table,
|
||||
backup,
|
||||
drop_site,
|
||||
install_app,
|
||||
|
|
|
|||
|
|
@ -22,6 +22,26 @@ frappe.ui.form.on("Recorder", {
|
|||
frm.reload_doc();
|
||||
setTimeout(() => frm.scroll_to_field("suggested_indexes"), 1500);
|
||||
});
|
||||
|
||||
let index_grid = frm.fields_dict.suggested_indexes.grid;
|
||||
index_grid.wrapper.find(".grid-footer").toggle(true);
|
||||
index_grid.toggle_checkboxes(true);
|
||||
index_grid.df.cannot_delete_rows = true;
|
||||
index_grid.add_custom_button(__("Add Indexes"), function () {
|
||||
let indexes_to_add = index_grid.get_selected_children().map((row) => {
|
||||
return {
|
||||
column: row.column,
|
||||
table: row.table,
|
||||
};
|
||||
});
|
||||
if (!indexes_to_add.length) {
|
||||
frappe.toast(__("You need to select indexes you want to add first."));
|
||||
return;
|
||||
}
|
||||
frappe.xcall("frappe.core.doctype.recorder.recorder.add_indexes", {
|
||||
indexes: indexes_to_add,
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
setup_sort: function (frm) {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
# Copyright (c) 2023, Frappe Technologies and contributors
|
||||
# For license information, please see license.txt
|
||||
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import frappe
|
||||
from frappe import _
|
||||
from frappe.core.doctype.recorder.db_optimizer import DBOptimizer, DBTable
|
||||
from frappe.custom.doctype.property_setter.property_setter import make_property_setter
|
||||
from frappe.model.document import Document
|
||||
from frappe.recorder import RECORDER_REQUEST_HASH
|
||||
from frappe.recorder import get as get_recorder_data
|
||||
|
|
@ -113,6 +115,34 @@ def serialize_request(request):
|
|||
return request
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def add_indexes(indexes):
|
||||
frappe.only_for("Administrator")
|
||||
indexes = json.loads(indexes)
|
||||
|
||||
for index in indexes:
|
||||
frappe.enqueue(_add_index, table=index["table"], column=index["column"])
|
||||
frappe.msgprint(_("Enqueued creation of indexes"), alert=True)
|
||||
|
||||
|
||||
def _add_index(table, column):
|
||||
doctype = get_doctype_name(table)
|
||||
frappe.db.add_index(doctype, [column])
|
||||
make_property_setter(
|
||||
doctype,
|
||||
column,
|
||||
property="search_index",
|
||||
value="1",
|
||||
property_type="Check",
|
||||
for_doctype=False, # Applied on docfield
|
||||
)
|
||||
frappe.msgprint(
|
||||
_("Index created successfully on column {0} of doctype {1}").format(column, doctype),
|
||||
alert=True,
|
||||
realtime=True,
|
||||
)
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def optimize(recorder_id: str):
|
||||
frappe.only_for("Administrator")
|
||||
|
|
@ -152,14 +182,18 @@ def _optimize(recorder_id):
|
|||
]
|
||||
|
||||
if not suggested_indexes:
|
||||
frappe.msgprint(_("No optimization suggestions."), realtime=True)
|
||||
frappe.msgprint(
|
||||
_("No automatic optimization suggestions available."),
|
||||
title=_("No Suggestions"),
|
||||
realtime=True,
|
||||
)
|
||||
return
|
||||
|
||||
frappe.msgprint(_("Query analysis complete. Check suggested indexes."), realtime=True, alert=True)
|
||||
data = frappe.cache.hget(RECORDER_REQUEST_HASH, record.name)
|
||||
data["suggested_indexes"] = [{"table": idx[0][0], "column": idx[0][1]} for idx in suggested_indexes]
|
||||
frappe.cache.hset(RECORDER_REQUEST_HASH, record.name, data)
|
||||
frappe.publish_realtime("recorder-analysis-complete")
|
||||
frappe.publish_realtime("recorder-analysis-complete", user=frappe.session.user)
|
||||
frappe.msgprint(_("Query analysis complete. Check suggested indexes."), realtime=True, alert=True)
|
||||
|
||||
|
||||
def _optimize_query(query):
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ import re
|
|||
|
||||
import frappe
|
||||
import frappe.recorder
|
||||
from frappe.core.doctype.recorder.recorder import serialize_request
|
||||
from frappe.core.doctype.recorder.recorder import _optimize_query, serialize_request
|
||||
from frappe.query_builder.utils import db_type_is
|
||||
from frappe.recorder import get as get_recorder_data
|
||||
from frappe.tests.test_query_builder import run_only_if
|
||||
from frappe.tests.utils import FrappeTestCase
|
||||
from frappe.utils import set_request
|
||||
|
||||
|
|
@ -75,3 +77,20 @@ class TestRecorder(FrappeTestCase):
|
|||
requests = frappe.get_all("Recorder")
|
||||
request_doc = get_recorder_data(requests[0].name)
|
||||
self.assertIsInstance(serialize_request(request_doc), dict)
|
||||
|
||||
|
||||
class TestQueryOptimization(FrappeTestCase):
|
||||
@run_only_if(db_type_is.MARIADB)
|
||||
def test_query_optimizer(self):
|
||||
suggested_index = _optimize_query(
|
||||
"""select name from
|
||||
`tabUser` u
|
||||
join `tabHas Role` r
|
||||
on r.parent = u.name
|
||||
where email='xyz'
|
||||
and creation > '2023'
|
||||
and bio like '%xyz%'
|
||||
"""
|
||||
)
|
||||
self.assertEqual(suggested_index.table, "tabUser")
|
||||
self.assertEqual(suggested_index.column, "email")
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
{
|
||||
"actions": [],
|
||||
"allow_rename": 1,
|
||||
"creation": "2024-05-14 15:03:46.138438",
|
||||
"creation": "2024-05-14 16:23:33.466465",
|
||||
"doctype": "DocType",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"field_order": [
|
||||
"table",
|
||||
"column",
|
||||
"add_index"
|
||||
"column"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
|
|
@ -22,19 +21,13 @@
|
|||
"fieldtype": "Data",
|
||||
"in_list_view": 1,
|
||||
"label": "Column"
|
||||
},
|
||||
{
|
||||
"columns": 2,
|
||||
"fieldname": "add_index",
|
||||
"fieldtype": "Button",
|
||||
"label": "Add Index"
|
||||
}
|
||||
],
|
||||
"index_web_pages_for_search": 1,
|
||||
"is_virtual": 1,
|
||||
"istable": 1,
|
||||
"links": [],
|
||||
"modified": "2024-05-14 15:18:51.371808",
|
||||
"modified": "2024-05-14 17:43:57.231051",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Core",
|
||||
"name": "Recorder Suggested Index",
|
||||
|
|
|
|||
|
|
@ -925,15 +925,6 @@ class TestDBUtils(BaseTestCommands):
|
|||
meta = frappe.get_meta("User", cached=False)
|
||||
self.assertTrue(meta.get_field(field).search_index)
|
||||
|
||||
@run_only_if(db_type_is.MARIADB)
|
||||
def test_describe_table(self):
|
||||
self.execute("bench --site {site} describe-database-table --doctype User", {})
|
||||
self.assertIn("user_type", self.stdout)
|
||||
|
||||
# Ensure that output is machine parseable
|
||||
stats = json.loads(self.stdout)
|
||||
self.assertIn("total_rows", stats)
|
||||
|
||||
|
||||
class TestSchedulerUtils(BaseTestCommands):
|
||||
# Retry just in case there are stuck queued jobs
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user