Merge branch 'develop' into email-account-form

.pre-commit-config.yaml

@@ -23,11 +23,15 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.0
     hooks:
+      - id: ruff
+        name: "Run ruff import sorter"
+        args: ["--select=I", "--fix"]
+
       - id: ruff
         name: "Run ruff linter"
 
       - id: ruff-format
-        name: "Format Python code"
+        name: "Run ruff formatter"
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.7.1

cypress.config.js

@@ -11,8 +11,8 @@ module.exports = defineConfig({
 	viewportHeight: 960,
 	viewportWidth: 1400,
 	retries: {
-		runMode: 2,
-		openMode: 2,
+		runMode: 1,
+		openMode: 1,
 	},
 	e2e: {
 		// We've imported your old cypress plugins here.

cypress/integration/discussions.js

@@ -1,85 +0,0 @@
-context("Discussions", () => {
-	before(() => {
-		cy.login();
-		cy.visit("/app");
-		return cy
-			.window()
-			.its("frappe")
-			.then((frappe) => {
-				return frappe.call("frappe.tests.ui_test_helpers.create_data_for_discussions");
-			});
-	});
-
-	const reply_through_modal = () => {
-		cy.visit("/test-page-discussions");
-
-		// Open the modal
-		cy.get(".reply").click();
-		cy.wait(500);
-		cy.get(".discussion-modal").should("be.visible");
-
-		// Enter title
-		cy.get(".modal .topic-title")
-			.type("Discussion from tests")
-			.should("have.value", "Discussion from tests");
-
-		// Enter comment
-		cy.get(".modal .discussions-comment").type(
-			"This is a discussion from the cypress ui tests."
-		);
-
-		// Submit
-		cy.get(".modal .submit-discussion").click();
-		cy.wait(2000);
-
-		// Check if discussion is added to page and content is visible
-		cy.get(".sidebar-parent:first .discussion-topic-title").should(
-			"have.text",
-			"Discussion from tests"
-		);
-		cy.get(".discussion-on-page:visible").should("have.class", "show");
-		cy.get(".discussion-on-page:visible .reply-card .reply-text .ql-editor p").should(
-			"have.text",
-			"This is a discussion from the cypress ui tests."
-		);
-	};
-
-	const reply_through_comment_box = () => {
-		cy.get(".discussion-form:visible .discussions-comment").type(
-			"This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page."
-		);
-
-		cy.get(".discussion-form:visible .submit-discussion").click();
-		cy.wait(3000);
-		cy.get(".discussion-on-page:visible").should("have.class", "show");
-		cy.get(".discussion-on-page:visible")
-			.children(".reply-card")
-			.eq(1)
-			.find(".reply-text")
-			.should(
-				"have.text",
-				"This is a discussion from the cypress ui tests. This comment was entered through the commentbox on the page.\n"
-			);
-	};
-
-	const single_thread_discussion = () => {
-		cy.visit("/test-single-thread");
-		cy.get(".discussions-sidebar").should("have.length", 0);
-		cy.get(".reply").should("have.length", 0);
-
-		cy.get(".discussion-form:visible .discussions-comment").type(
-			"This comment is being made on a single thread discussion."
-		);
-		cy.get(".discussion-form:visible .submit-discussion").click();
-		cy.wait(3000);
-		cy.get(".discussion-on-page")
-			.children(".reply-card")
-			.eq(-1)
-			.find(".reply-text")
-			.should("have.text", "This comment is being made on a single thread discussion.\n");
-	};
-
-	it("reply through modal", reply_through_modal);
-	it("reply through comment box", reply_through_comment_box);
-	it("single thread discussion", single_thread_discussion);
-});

cypress/integration/first_day_of_the_week.js

@@ -1,51 +0,0 @@
-context("First Day of the Week", () => {
-	before(() => {
-		cy.login();
-	});
-
-	beforeEach(() => {
-		cy.visit("/app/system-settings");
-		cy.findByText("Date and Number Format").click();
-	});
-
-	it("Date control starts with same day as selected in System Settings", () => {
-		cy.intercept(
-			"POST",
-			"/api/method/frappe.core.doctype.system_settings.system_settings.load"
-		).as("load_settings");
-		cy.fill_field("first_day_of_the_week", "Tuesday", "Select");
-		cy.findByRole("button", { name: "Save" }).click();
-		cy.wait("@load_settings");
-		cy.dialog({
-			title: "Date",
-			fields: [
-				{
-					label: "Date",
-					fieldname: "date",
-					fieldtype: "Date",
-				},
-			],
-		});
-		cy.get_field("date").click();
-		cy.get(".datepicker--day-name").eq(0).should("have.text", "Tu");
-	});
-
-	it("Calendar view starts with same day as selected in System Settings", () => {
-		cy.intercept(
-			"POST",
-			"/api/method/frappe.core.doctype.system_settings.system_settings.load"
-		).as("load_settings");
-		cy.fill_field("first_day_of_the_week", "Monday", "Select");
-		cy.findByRole("button", { name: "Save" }).click();
-		cy.wait("@load_settings");
-		cy.visit("app/todo/view/calendar/default");
-		cy.get(".fc-day-header > span").eq(0).should("have.text", "Mon");
-	});
-
-	after(() => {
-		cy.visit("/app/system-settings");
-		cy.findByText("Date and Number Format").click();
-		cy.fill_field("first_day_of_the_week", "Sunday", "Select");
-		cy.findByRole("button", { name: "Save" }).click();
-	});
-});

cypress/integration/folder_navigation.js

@@ -1,97 +0,0 @@
-context("Folder Navigation", () => {
-	before(() => {
-		cy.visit("/login");
-		cy.login();
-		cy.visit("/app/file");
-	});
-
-	it("Adding Folders", () => {
-		//Adding filter to go into the home folder
-		cy.get(".filter-x-button").click();
-		cy.click_filter_button();
-		cy.get(".filter-action-buttons > .text-muted").findByText("+ Add a Filter").click();
-		cy.get(".fieldname-select-area > .awesomplete > .form-control:last").type("Fol{enter}");
-		cy.get(".filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback")
-			.first()
-			.type("Home{enter}");
-		cy.get(".filter-action-buttons > div > .btn-primary").findByText("Apply Filters").click();
-
-		//Adding folder (Test Folder)
-		cy.click_menu_button("New Folder");
-		cy.fill_field("value", "Test Folder");
-		cy.click_modal_primary_button("Create");
-	});
-
-	it("Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct", () => {
-		//Navigating inside the Attachments folder
-		cy.clear_filters();
-		cy.wait(500);
-		cy.get('[title="Attachments"] > span').click();
-
-		//To check if the URL formed after visiting the attachments folder is correct
-		cy.location("pathname").should("eq", "/app/file/view/home/Attachments");
-		cy.visit("/app/file/view/home/Attachments");
-
-		//Adding folder inside the attachments folder
-		cy.click_menu_button("New Folder");
-		cy.fill_field("value", "Test Folder");
-		cy.click_modal_primary_button("Create");
-
-		//Navigating inside the added folder in the Attachments folder
-		cy.wait(500);
-		cy.get('[title="Test Folder"] > span').click();
-
-		//To check if the URL is correct after visiting the Test Folder
-		cy.location("pathname").should("eq", "/app/file/view/home/Attachments/Test%20Folder");
-		cy.visit("/app/file/view/home/Attachments/Test%20Folder");
-
-		//Adding a file inside the Test Folder
-		cy.findByRole("button", { name: "Add File" }).eq(0).click({ force: true });
-		cy.get(".file-uploader").findByText("Link").click();
-		cy.get(".input-group > input.form-control:visible").as("upload_input");
-		cy.get("@upload_input").type("https://wallpaperplay.com/walls/full/8/2/b/72402.jpg", {
-			waitForAnimations: false,
-			parseSpecialCharSequences: false,
-			force: true,
-			delay: 100,
-		});
-		cy.click_modal_primary_button("Upload");
-
-		//To check if the added file is present in the Test Folder
-		cy.visit("/app/file/view/home/Attachments");
-		cy.wait(500);
-		cy.get("span.level-item > a > span").should("contain", "Test Folder");
-		cy.visit("/app/file/view/home/Attachments/Test%20Folder");
-
-		cy.wait(500);
-		cy.get(".list-row-container").eq(1).should("contain.text", "72402.jpg");
-		cy.get(".list-row-checkbox").eq(1).click();
-
-		cy.intercept({
-			method: "POST",
-			url: "api/method/frappe.desk.reportview.delete_items",
-		}).as("file_deleted");
-
-		//Deleting the added file from the Test folder
-		cy.click_action_button("Delete");
-		cy.click_modal_primary_button("Yes");
-		cy.wait("@file_deleted");
-
-		//Deleting the Test Folder
-		cy.visit("/app/file/view/home/Attachments");
-		cy.get(".list-row-checkbox").eq(0).click();
-		cy.click_action_button("Delete");
-		cy.click_modal_primary_button("Yes");
-		cy.wait("@file_deleted");
-	});
-
-	it("Deleting Test Folder from the home", () => {
-		//Deleting the Test Folder added in the home directory
-		cy.visit("/app/file/view/home");
-		cy.get(".level-left > .list-subject > .file-select >.list-row-checkbox")
-			.eq(0)
-			.click({ force: true, delay: 500 });
-		cy.click_action_button("Delete");
-		cy.click_modal_primary_button("Yes");
-	});
-});

cypress/integration/list_view_drag_select.js

@@ -1,49 +0,0 @@
-context("List View", () => {
-	before(() => {
-		cy.login();
-		cy.go_to_list("DocType");
-	});
-
-	it("List view check rows on drag", () => {
-		cy.get(".filter-x-button").click();
-		cy.get(".list-row-checkbox").then(($checkbox) => {
-			cy.wrap($checkbox).first().trigger("mousedown");
-			cy.get(".level.list-row").each(($ele) => {
-				cy.wrap($ele).trigger("mousemove");
-			});
-			cy.document().trigger("mouseup");
-		});
-
-		cy.get(".level.list-row .list-row-checkbox").each(($checkbox) => {
-			cy.wrap($checkbox).should("be.checked");
-		});
-	});
-
-	it("Check all rows are checked", () => {
-		cy.get(".level.list-row .list-row-checkbox")
-			.its("length")
-			.then((len) => {
-				cy.get(".level-item.list-header-meta")
-					.should("be.visible")
-					.should("contain.text", `${len} items selected`);
-			});
-	});
-
-	it("List view uncheck rows on drag", () => {
-		cy.get(".list-row-checkbox").then(($checkbox) => {
-			cy.wrap($checkbox).first().trigger("mousedown");
-			cy.get(".level.list-row").each(($ele) => {
-				cy.wrap($ele).trigger("mousemove");
-			});
-			cy.document().trigger("mouseup");
-		});
-
-		cy.get(".level.list-row .list-row-checkbox").each(($checkbox) => {
-			cy.wrap($checkbox).should("not.be.checked");
-		});
-	});
-
-	it("Check all rows are unchecked", () => {
-		cy.get(".level-item.list-header-meta").should("not.be.visible");
-	});
-});

cypress/integration/workspace_blocks.js

@@ -166,7 +166,7 @@ context("Workspace Blocks", () => {
 
 		// add number card
 		cy.fill_field("number_card_name", "Test Number Card", "Link");
-		cy.get('[data-fieldname="number_card_name"] ul li').contains("Test Number Card").click();
+		cy.get('[data-fieldname="number_card_name"] ul div').contains("Test Number Card").click();
 		cy.click_modal_primary_button("Add");
 		cy.get(".ce-block .number-widget-box").first().as("number_card");
 		cy.get("@number_card").find(".widget-title").should("contain", "Test Number Card");

frappe/__init__.py

@@ -20,6 +20,7 @@ import json
 import os
 import re
 import signal
+import sys
 import traceback
 import warnings
 from collections.abc import Callable
@@ -541,9 +542,7 @@ def log(msg: str) -> None:
 	"""Add to `debug_log`
 
 	:param msg: Message."""
-	if not request:
-		print(repr(msg))
-
+	print(msg, file=sys.stderr)
 	debug_log.append(as_unicode(msg))
 
 
@@ -583,7 +582,6 @@ def msgprint(
 	:param realtime: Publish message immediately using websocket.
 	"""
 	import inspect
-	import sys
 
 	msg = safe_decode(msg)
 	out = _dict(message=msg)
@@ -2501,9 +2499,22 @@ def safe_encode(param, encoding="utf-8"):
 	return param
 
 
-def safe_decode(param, encoding="utf-8"):
+def safe_decode(param, encoding="utf-8", fallback_map: dict | None = None):
+	"""
+	Method to safely decode data into a string
+
+	:param param: The data to be decoded
+	:param encoding: The encoding to decode into
+	:param fallback_map: A fallback map to reference in case of a LookupError
+	:return:
+	"""
 	try:
 		param = param.decode(encoding)
+	except LookupError:
+		try:
+			param = param.decode((fallback_map or {}).get(encoding, "utf-8"))
+		except Exception:
+			pass
 	except Exception:
 		pass
 	return param
@@ -2551,7 +2562,11 @@ def validate_and_sanitize_search_inputs(fn):
 
 
 def _register_fault_handler():
-	faulthandler.register(signal.SIGUSR1)
+	import io
+
+	# Some libraries monkey patch stderr, we need actual fd
+	if isinstance(sys.stderr, io.TextIOWrapper):
+		faulthandler.register(signal.SIGUSR1, file=sys.stderr)
 
 
 from frappe.utils.error import log_error

frappe/core/doctype/doctype/doctype.py

@@ -1030,7 +1030,7 @@ class DocType(Document):
 
 		file = Path(get_file_path(frappe.scrub(self.module), self.doctype, self.name))
 		content = json.loads(file.read_text())
-		if content.get("modified") and get_datetime(self.modified) != get_datetime(content.get("modified")):
+		if content.get("modified") and get_datetime(self.modified) < get_datetime(content.get("modified")):
 			frappe.msgprint(
 				_(
 					"This doctype has pending migrations, run 'bench migrate' before modifying the doctype to avoid losing changes."

frappe/core/doctype/file/utils.py

@@ -25,7 +25,7 @@ if TYPE_CHECKING:
 
 def make_home_folder() -> None:
 	home = frappe.get_doc(
-		{"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")}
+		{"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": "Home"}
 	).insert(ignore_if_duplicate=True)
 
 	frappe.get_doc(
@@ -34,7 +34,7 @@ def make_home_folder() -> None:
 			"folder": home.name,
 			"is_folder": 1,
 			"is_attachments_folder": 1,
-			"file_name": _("Attachments"),
+			"file_name": "Attachments",
 		}
 	).insert(ignore_if_duplicate=True)
 
@@ -361,15 +361,15 @@ def attach_files_to_document(doc: "Document", event) -> None:
 
 
 def relink_files(doc, fieldname, temp_doc_name):
-	if not temp_doc_name:
-		return
-	from frappe.utils.data import add_to_date, now_datetime
-
 	"""
 	Relink files attached to incorrect document name to the new document name
 	by check if file with temp name exists that was created in last 60 minutes
 	"""
-	mislinked_file = frappe.db.exists(
+	if not temp_doc_name:
+		return
+	from frappe.utils.data import add_to_date, now_datetime
+
+	mislinked_file = frappe.db.get_value(
 		"File",
 		{
 			"file_url": doc.get(fieldname),
@@ -382,7 +382,7 @@ def relink_files(doc, fieldname, temp_doc_name):
 			),
 		},
 	)
-	"""If file exists, attach it to the new docname"""
+	# If file exists, attach it to the new docname
 	if mislinked_file:
 		frappe.db.set_value(
 			"File",

frappe/core/doctype/prepared_report/prepared_report.js

@@ -41,14 +41,8 @@ frappe.ui.form.on("Prepared Report", {
 
 		if (frm.doc.status == "Completed") {
 			frm.page.set_primary_action(__("Show Report"), () => {
-				frappe.route_options = filters;
-				frappe.set_route(
-					"query-report",
-					frm.doc.report_name,
-					frappe.utils.make_query_string({
-						prepared_report_name: frm.doc.name,
-					})
-				);
+				frappe.route_options = { prepared_report_name: frm.doc.name };
+				frappe.set_route("query-report", frm.doc.report_name);
 			});
 		}
 	},

frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.js

@@ -3,4 +3,49 @@
 
 frappe.query_reports["Database Storage Usage By Tables"] = {
 	filters: [],
+	onload: function (report) {
+		report.page.add_inner_button(
+			__("Optimize"),
+			function () {
+				let d = new frappe.ui.Dialog({
+					title: "Optimize Doctype",
+					fields: [
+						{
+							label: "Select a DocType",
+							fieldname: "doctype_name",
+							fieldtype: "Link",
+							options: "DocType",
+							get_query: function () {
+								return {
+									filters: { issingle: ["=", false], is_virtual: ["=", false] },
+								};
+							},
+						},
+					],
+					size: "small",
+					primary_action_label: "Optimize",
+					primary_action(values) {
+						frappe.call({
+							method: "frappe.core.report.database_storage_usage_by_tables.database_storage_usage_by_tables.optimize_doctype",
+							args: {
+								doctype_name: values.doctype_name,
+							},
+							callback: function (r) {
+								if (!r.exec) {
+									frappe.show_alert(
+										__(
+											`${values.doctype_name} has been added to queue for optimization`
+										)
+									);
+								}
+							},
+						});
+						d.hide();
+					},
+				});
+				d.show();
+			},
+			__("Actions")
+		);
+	},
 };

frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.py

@@ -38,3 +38,27 @@ def execute(filters=None):
 		as_dict=1,
 	)
 	return COLUMNS, data
+
+
+@frappe.whitelist()
+def optimize_doctype(doctype_name: str):
+	frappe.only_for("System Manager")
+	frappe.enqueue(
+		optimize_doctype_job,
+		queue="long",
+		job_id=f"optimize-{doctype_name}",
+		doctype_name=doctype_name,
+		deduplicate=True,
+	)
+
+
+def optimize_doctype_job(doctype_name: str):
+	from frappe.utils import get_table_name
+
+	doctype_table = get_table_name(doctype_name, wrap_in_backticks=True)
+	if frappe.db.db_type == "mariadb":
+		query = f"OPTIMIZE TABLE {doctype_table};"
+	else:
+		query = f"VACUUM (ANALYZE) {doctype_table};"
+
+	frappe.db.sql(query)

frappe/desk/doctype/number_card/number_card.js

@@ -319,7 +319,7 @@ frappe.ui.form.on("Number Card", {
 	},
 
 	render_dynamic_filters_table(frm) {
-		if (!frappe.boot.developer_mode || !frm.doc.is_standard || frm.doc.type == "Custom") {
+		if (frm.doc.type == "Custom") {
 			return;
 		}
 

frappe/desk/doctype/system_health_report/system_health_report.js

@@ -0,0 +1,98 @@
+// Copyright (c) 2024, Frappe Technologies and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("System Health Report", {
+	onload(frm) {
+		let poll_attempts = 0;
+		const interval = setInterval(() => {
+			frappe
+				.xcall(
+					"frappe.desk.doctype.system_health_report.system_health_report.get_job_status",
+					{ job_id: frm.doc.test_job_id }
+				)
+				.then((status) => {
+					poll_attempts += 1;
+					if (["finished", "failed"].includes(status) || poll_attempts > 30) {
+						clearInterval(interval);
+					}
+					status && frm.set_value("background_jobs_check", status);
+				});
+		}, 1000);
+	},
+	refresh(frm) {
+		frm.set_value("socketio_ping_check", "Fail");
+		frappe.realtime.on("pong", () => {
+			frm.set_value("socketio_ping_check", "Pass");
+			frm.set_value(
+				"socketio_transport_mode",
+				frappe.realtime.socket.io?.engine?.transport?.name
+			);
+		});
+		frappe.realtime.emit("ping");
+		frm.disable_save();
+		frm.trigger("setup_highlight");
+	},
+
+	setup_highlight(frm) {
+		/// field => is bad?
+		const conditions = {
+			scheduler_status: (val) => val.toLowerCase() != "active",
+			background_jobs_check: (val) => val.toLowerCase() != "finished",
+			total_background_workers: (val) => val == 0,
+			binary_logging: (val) => val.toLowerCase() != "on",
+			socketio_ping_check: (val) => val != "Pass",
+			socketio_transport_mode: (val) => val != "websocket",
+			onsite_backups: (val) => val == 0,
+			failed_logins: (val) => val > frm.doc.total_users,
+			total_errors: (val) => val > 50,
+			// 5% excluding very small numbers
+			unhandled_emails: (val) =>
+				val > 3 && frm.doc.handled_emails > 3 && val / frm.doc.handled_emails > 0.05,
+			failed_emails: (val) =>
+				val > 3 &&
+				frm.doc.total_outgoing_emails > 3 &&
+				val / frm.doc.total_outgoing_emails > 0.05,
+			pending_emails: (val) =>
+				val > 3 &&
+				frm.doc.total_outgoing_emails > 3 &&
+				val / frm.doc.total_outgoing_emails > 0.1,
+			"queue_status.pending_jobs": (val) => val > 50,
+			"background_workers.utilization": (val) => val > 70,
+			"background_workers.failed_jobs": (val) => val > 50,
+			"top_errors.occurrences": (val) => val > 10,
+			"failing_scheduled_jobs.failure_rate": (val) => val > 10,
+		};
+
+		const style = document.createElement("style");
+		style.innerText = `.health-check-failed {
+				font-weight: bold;
+				color: var(--text-colour);
+				background-color: var(--bg-red);
+			}`;
+		document.head.appendChild(style);
+
+		const update_fields = () => {
+			Object.entries(conditions).forEach(([field, condition]) => {
+				try {
+					if (field.includes(".")) {
+						let [table, fieldname] = field.split(".");
+
+						frm.fields_dict[table].grid.grid_rows.forEach((row) => {
+							let is_bad = condition(row.doc[fieldname]);
+							$(row.columns[fieldname]).toggleClass("health-check-failed", is_bad);
+						});
+					} else {
+						let is_bad = condition(frm.doc[field]);
+						let df = frm.fields_dict[field];
+						$(df.disp_area).toggleClass("health-check-failed", is_bad);
+					}
+				} catch (e) {
+					console.log("Failed to evaluated", e);
+				}
+			});
+		};
+
+		update_fields();
+		setInterval(update_fields, 1000);
+	},
+});

frappe/desk/doctype/system_health_report/system_health_report.json

@@ -0,0 +1,400 @@
+{
+ "actions": [],
+ "beta": 1,
+ "creation": "2024-04-18 16:59:15.088271",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "background_jobs_tab",
+  "background_jobs_section",
+  "total_background_workers",
+  "column_break_klex",
+  "background_jobs_check",
+  "test_job_id",
+  "section_break_djoz",
+  "queue_status",
+  "column_break_wjoz",
+  "background_workers",
+  "scheduler_section",
+  "scheduler_status",
+  "failing_scheduled_jobs",
+  "database_section",
+  "database",
+  "database_version",
+  "db_storage_usage",
+  "column_break_auhv",
+  "top_db_tables",
+  "mariadb_variables_section",
+  "bufferpool_size",
+  "column_break_vztw",
+  "binary_logging",
+  "cache_section",
+  "cache_keys",
+  "column_break_ccov",
+  "cache_memory_usage",
+  "realtime_socketio_section",
+  "socketio_ping_check",
+  "column_break_hgay",
+  "socketio_transport_mode",
+  "section_break_ryki",
+  "storage_usage_column",
+  "public_files_size",
+  "column_break_jnkt",
+  "private_files_size",
+  "backups_section",
+  "backups_column",
+  "onsite_backups",
+  "column_break_luox",
+  "backups_size",
+  "users_section",
+  "total_users",
+  "new_users",
+  "failed_logins",
+  "active_sessions",
+  "column_break_yfwd",
+  "last_10_active_users",
+  "section_break_udjs",
+  "outgoing_emails_column",
+  "total_outgoing_emails",
+  "pending_emails",
+  "failed_emails",
+  "incoming_emails_last_7_days_column",
+  "handled_emails",
+  "unhandled_emails",
+  "errors_generated_in_last_1_day_section",
+  "total_errors",
+  "top_errors",
+  "column_break_fzke"
+ ],
+ "fields": [
+  {
+   "fieldname": "background_workers",
+   "fieldtype": "Table",
+   "label": "Background Workers",
+   "options": "System Health Report Workers"
+  },
+  {
+   "documentation_url": "/app/rq-worker",
+   "fieldname": "total_background_workers",
+   "fieldtype": "Int",
+   "label": "Total Background Workers"
+  },
+  {
+   "fieldname": "background_jobs_section",
+   "fieldtype": "Section Break",
+   "hide_border": 1,
+   "label": "Background Jobs"
+  },
+  {
+   "documentation_url": "/app/rq-job",
+   "fieldname": "scheduler_status",
+   "fieldtype": "Data",
+   "label": "Scheduler Status"
+  },
+  {
+   "fieldname": "queue_status",
+   "fieldtype": "Table",
+   "label": "Queue Status",
+   "options": "System Health Report Queue"
+  },
+  {
+   "fieldname": "column_break_klex",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_djoz",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "column_break_wjoz",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "background_jobs_tab",
+   "fieldtype": "Tab Break",
+   "label": "Report"
+  },
+  {
+   "default": "Fail",
+   "fieldname": "socketio_ping_check",
+   "fieldtype": "Select",
+   "label": "SocketIO Ping Check",
+   "options": "Fail\nPass"
+  },
+  {
+   "fieldname": "column_break_hgay",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "socketio_transport_mode",
+   "fieldtype": "Select",
+   "label": "SocketIO Transport Mode",
+   "options": "Polling\nWebsocket"
+  },
+  {
+   "fieldname": "outgoing_emails_column",
+   "fieldtype": "Column Break",
+   "label": "Outgoing Emails (Last 7 days)"
+  },
+  {
+   "documentation_url": "/app/email-queue?status=Error",
+   "fieldname": "failed_emails",
+   "fieldtype": "Int",
+   "label": "Failed Emails"
+  },
+  {
+   "fieldname": "total_outgoing_emails",
+   "fieldtype": "Int",
+   "label": "Total Outgoing Emails"
+  },
+  {
+   "documentation_url": "/app/email-queue?status=Not+Sent",
+   "fieldname": "pending_emails",
+   "fieldtype": "Int",
+   "label": "Pending Emails"
+  },
+  {
+   "fieldname": "incoming_emails_last_7_days_column",
+   "fieldtype": "Column Break",
+   "label": "Incoming Emails (Last 7 days)"
+  },
+  {
+   "documentation_url": "/app/unhandled-email",
+   "fieldname": "unhandled_emails",
+   "fieldtype": "Int",
+   "label": "Unhandled Emails"
+  },
+  {
+   "documentation_url": "/app/communication?communication_type=Communication&sent_or_received=Received",
+   "fieldname": "handled_emails",
+   "fieldtype": "Int",
+   "label": "Handled Emails"
+  },
+  {
+   "fieldname": "errors_generated_in_last_1_day_section",
+   "fieldtype": "Section Break",
+   "label": "Errors"
+  },
+  {
+   "documentation_url": "/app/error-log",
+   "fieldname": "total_errors",
+   "fieldtype": "Int",
+   "label": "Total Errors (last 1 day)"
+  },
+  {
+   "fieldname": "top_errors",
+   "fieldtype": "Table",
+   "label": "Top Errors",
+   "options": "System Health Report Errors"
+  },
+  {
+   "fieldname": "database",
+   "fieldtype": "Data",
+   "label": "Database"
+  },
+  {
+   "fieldname": "db_storage_usage",
+   "fieldtype": "Float",
+   "label": "Storage Usage (MB)"
+  },
+  {
+   "fieldname": "column_break_auhv",
+   "fieldtype": "Column Break"
+  },
+  {
+   "documentation_url": "/app/query-report/Database Storage Usage By Tables",
+   "fieldname": "top_db_tables",
+   "fieldtype": "Table",
+   "label": "Storage Usage By Table",
+   "options": "System Health Report Tables"
+  },
+  {
+   "fieldname": "database_version",
+   "fieldtype": "Data",
+   "label": "Database Version"
+  },
+  {
+   "fieldname": "mariadb_variables_section",
+   "fieldtype": "Section Break",
+   "label": "MariaDB Variables"
+  },
+  {
+   "fieldname": "bufferpool_size",
+   "fieldtype": "Data",
+   "label": "Bufferpool Size"
+  },
+  {
+   "fieldname": "binary_logging",
+   "fieldtype": "Data",
+   "label": "Binary Logging"
+  },
+  {
+   "fieldname": "cache_keys",
+   "fieldtype": "Int",
+   "label": "Number of keys"
+  },
+  {
+   "fieldname": "cache_memory_usage",
+   "fieldtype": "Data",
+   "label": "Memory Usage"
+  },
+  {
+   "fieldname": "backups_size",
+   "fieldtype": "Float",
+   "label": "Backups (MB)"
+  },
+  {
+   "fieldname": "private_files_size",
+   "fieldtype": "Float",
+   "label": "Private Files (MB)"
+  },
+  {
+   "fieldname": "public_files_size",
+   "fieldtype": "Float",
+   "label": "Public Files (MB)"
+  },
+  {
+   "fieldname": "storage_usage_column",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "backups_column",
+   "fieldtype": "Column Break"
+  },
+  {
+   "documentation_url": "/app/backups",
+   "fieldname": "onsite_backups",
+   "fieldtype": "Int",
+   "label": "Number of onsite backups"
+  },
+  {
+   "documentation_url": "/app/user",
+   "fieldname": "total_users",
+   "fieldtype": "Int",
+   "label": "Total Users"
+  },
+  {
+   "fieldname": "column_break_yfwd",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "new_users",
+   "fieldtype": "Int",
+   "label": "New Users (Last 30 days)"
+  },
+  {
+   "documentation_url": "/app/activity-log?status=Failed&operation=Login",
+   "fieldname": "failed_logins",
+   "fieldtype": "Int",
+   "label": "Failed Logins (Last 30 days)"
+  },
+  {
+   "fieldname": "active_sessions",
+   "fieldtype": "Int",
+   "label": "Active Sessions"
+  },
+  {
+   "fieldname": "last_10_active_users",
+   "fieldtype": "Code",
+   "label": "Last 10 active users"
+  },
+  {
+   "fieldname": "section_break_udjs",
+   "fieldtype": "Section Break",
+   "label": "Emails"
+  },
+  {
+   "fieldname": "database_section",
+   "fieldtype": "Section Break",
+   "label": "Database"
+  },
+  {
+   "fieldname": "cache_section",
+   "fieldtype": "Section Break",
+   "label": "Cache"
+  },
+  {
+   "fieldname": "column_break_ccov",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_ryki",
+   "fieldtype": "Section Break",
+   "label": "File Storage"
+  },
+  {
+   "fieldname": "column_break_vztw",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_jnkt",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "backups_section",
+   "fieldtype": "Section Break",
+   "label": "Backups"
+  },
+  {
+   "fieldname": "column_break_luox",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "users_section",
+   "fieldtype": "Section Break",
+   "label": "Users"
+  },
+  {
+   "fieldname": "realtime_socketio_section",
+   "fieldtype": "Section Break",
+   "label": "Realtime (SocketIO)"
+  },
+  {
+   "documentation_url": "/app/rq-job",
+   "fieldname": "background_jobs_check",
+   "fieldtype": "Data",
+   "label": "Background Jobs Check"
+  },
+  {
+   "fieldname": "test_job_id",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Test Job ID"
+  },
+  {
+   "fieldname": "column_break_fzke",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "scheduler_section",
+   "fieldtype": "Section Break",
+   "label": "Scheduler"
+  },
+  {
+   "fieldname": "failing_scheduled_jobs",
+   "fieldtype": "Table",
+   "label": "Failing Scheduled Jobs (last 7 days)",
+   "options": "System Health Report Failing Jobs"
+  }
+ ],
+ "hide_toolbar": 1,
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2024-04-22 11:47:52.194784",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "print": 1,
+   "read": 1,
+   "role": "System Manager"
+  }
+ ],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report/system_health_report.py

@@ -0,0 +1,320 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+"""
+Basic system health check report to see how everything on site is functioning in one single page.
+
+Metrics:
+- Background jobs, workers and scheduler summary, queue stats
+- SocketIO works (using basic ping test)
+- Email queue flush and pull
+- Error logs status
+- Database - storage usage and top tables, version
+- Cache
+- Storage - files usage
+- Backups
+- User - new users, sessions stats, failed login attempts
+"""
+
+import functools
+import os
+from collections import defaultdict
+from collections.abc import Callable
+
+import frappe
+from frappe.model.document import Document
+from frappe.utils.background_jobs import get_queue, get_queue_list
+from frappe.utils.caching import redis_cache
+from frappe.utils.data import add_to_date
+from frappe.utils.scheduler import get_scheduler_status
+
+
+def health_check(step: str):
+	assert isinstance(step, str), "Invalid usage of decorator, Usage: @health_check('step name')"
+
+	def suppress_exception(func: Callable):
+		@functools.wraps(func)
+		def wrapper(*args, **kwargs):
+			try:
+				return func(*args, **kwargs)
+			except Exception as e:
+				# nosemgrep
+				frappe.msgprint(f"System Health check step {frappe.bold(step)} failed: {e}", alert=True)
+
+		return wrapper
+
+	return suppress_exception
+
+
+class SystemHealthReport(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.desk.doctype.system_health_report_errors.system_health_report_errors import (
+			SystemHealthReportErrors,
+		)
+		from frappe.desk.doctype.system_health_report_failing_jobs.system_health_report_failing_jobs import (
+			SystemHealthReportFailingJobs,
+		)
+		from frappe.desk.doctype.system_health_report_queue.system_health_report_queue import (
+			SystemHealthReportQueue,
+		)
+		from frappe.desk.doctype.system_health_report_tables.system_health_report_tables import (
+			SystemHealthReportTables,
+		)
+		from frappe.desk.doctype.system_health_report_workers.system_health_report_workers import (
+			SystemHealthReportWorkers,
+		)
+		from frappe.types import DF
+
+		active_sessions: DF.Int
+		background_jobs_check: DF.Data | None
+		background_workers: DF.Table[SystemHealthReportWorkers]
+		backups_size: DF.Float
+		binary_logging: DF.Data | None
+		bufferpool_size: DF.Data | None
+		cache_keys: DF.Int
+		cache_memory_usage: DF.Data | None
+		database: DF.Data | None
+		database_version: DF.Data | None
+		db_storage_usage: DF.Float
+		failed_emails: DF.Int
+		failed_logins: DF.Int
+		failing_scheduled_jobs: DF.Table[SystemHealthReportFailingJobs]
+		handled_emails: DF.Int
+		last_10_active_users: DF.Code | None
+		new_users: DF.Int
+		onsite_backups: DF.Int
+		pending_emails: DF.Int
+		private_files_size: DF.Float
+		public_files_size: DF.Float
+		queue_status: DF.Table[SystemHealthReportQueue]
+		scheduler_status: DF.Data | None
+		socketio_ping_check: DF.Literal["Fail", "Pass"]
+		socketio_transport_mode: DF.Literal["Polling", "Websocket"]
+		test_job_id: DF.Data | None
+		top_db_tables: DF.Table[SystemHealthReportTables]
+		top_errors: DF.Table[SystemHealthReportErrors]
+		total_background_workers: DF.Int
+		total_errors: DF.Int
+		total_outgoing_emails: DF.Int
+		total_users: DF.Int
+		unhandled_emails: DF.Int
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		super(Document, self).__init__({})
+		frappe.only_for("System Manager")
+
+		# Each method loads a section of health report
+		# They should be written in a manner they are least likely to fail and if they do fail,
+		# they should indicate that in UI.
+		# This is best done by initializing fields with values that indicate that we haven't yet
+		# fetched the values.
+		self.fetch_background_jobs()
+		self.fetch_scheduler()
+		self.fetch_email_stats()
+		self.fetch_errors()
+		self.fetch_database_details()
+		self.fetch_cache_details()
+		self.fetch_storage_details()
+		self.fetch_user_stats()
+
+	@health_check("Background Jobs")
+	def fetch_background_jobs(self):
+		self.test_job_id = frappe.enqueue("frappe.ping", at_front=True).id
+		self.background_jobs_check = "queued"
+		self.scheduler_status = get_scheduler_status().get("status")
+		workers = frappe.get_all("RQ Worker")
+		self.total_background_workers = len(workers)
+		queue_summary = defaultdict(list)
+
+		for worker in workers:
+			queue_summary[worker.queue_type].append(worker)
+
+		for queue_type, workers in queue_summary.items():
+			self.append(
+				"background_workers",
+				{
+					"count": len(workers),
+					"queues": queue_type,
+					"failed_jobs": sum(w.failed_job_count for w in workers),
+					"utilization": sum(w.utilization_percent for w in workers) / len(workers),
+				},
+			)
+
+		for queue in get_queue_list():
+			q = get_queue(queue)
+			self.append(
+				"queue_status",
+				{
+					"queue": queue,
+					"pending_jobs": q.count,
+				},
+			)
+
+	@health_check("Scheduler")
+	def fetch_scheduler(self):
+		lower_threshold = add_to_date(None, days=-7, as_datetime=True)
+		# Exclude "maybe" curently executing job
+		upper_threshold = add_to_date(None, minutes=-30, as_datetime=True)
+		self.scheduler_status = get_scheduler_status().get("status")
+		failing_jobs = frappe.db.sql(
+			"""
+			select scheduled_job_type,
+				   avg(CASE WHEN status != 'Complete' THEN 1 ELSE 0 END) * 100 as failure_rate
+			from `tabScheduled Job Log`
+			where
+				creation > %(lower_threshold)s
+				and modified > %(lower_threshold)s
+				and creation < %(upper_threshold)s
+			group by scheduled_job_type
+			having failure_rate > 0
+			order by failure_rate desc
+			limit 5""",
+			{"lower_threshold": lower_threshold, "upper_threshold": upper_threshold},
+			as_dict=True,
+		)
+
+		for job in failing_jobs:
+			self.append("failing_scheduled_jobs", job)
+
+	@health_check("Emails")
+	def fetch_email_stats(self):
+		threshold = add_to_date(None, days=-7, as_datetime=True)
+		filters = {"creation": (">", threshold), "modified": (">", threshold)}
+		self.total_outgoing_emails = frappe.db.count("Email Queue", filters)
+		self.pending_emails = frappe.db.count("Email Queue", {"status": "Not Sent", **filters})
+		self.failed_emails = frappe.db.count("Email Queue", {"status": "Error", **filters})
+		self.unhandled_emails = frappe.db.count("Unhandled Email", filters)
+		self.handled_emails = frappe.db.count(
+			"Communication",
+			{"sent_or_received": "Received", "communication_type": "Communication", **filters},
+		)
+
+	@health_check("Errors")
+	def fetch_errors(self):
+		threshold = add_to_date(None, days=-1, as_datetime=True)
+		filters = {"creation": (">", threshold), "modified": (">", threshold)}
+		self.total_errors = frappe.db.count("Error Log", filters)
+
+		top_errors = frappe.db.sql(
+			"""select method as title, count(*) as occurrences
+			from `tabError Log`
+			where modified > %(threshold)s and creation > %(threshold)s
+			group by method
+			order by occurrences desc
+			limit 5""",
+			{"threshold": threshold},
+			as_dict=True,
+		)
+		for row in top_errors:
+			self.append("top_errors", row)
+
+	@health_check("Database")
+	def fetch_database_details(self):
+		from frappe.core.report.database_storage_usage_by_tables.database_storage_usage_by_tables import (
+			execute as db_report,
+		)
+
+		_cols, data = db_report()
+		self.database = frappe.db.db_type
+		self.db_storage_usage = sum(table.size for table in data)
+		for row in data[:5]:
+			self.append("top_db_tables", row)
+		self.database_version = frappe.db.sql("select version()")[0][0]
+
+		if frappe.db.db_type == "mariadb":
+			self.bufferpool_size = frappe.db.sql("show variables like 'innodb_buffer_pool_size'")[0][1]
+			self.binary_logging = frappe.db.sql("show variables like 'log_bin'")[0][1]
+
+	@health_check("Cache")
+	def fetch_cache_details(self):
+		self.cache_keys = len(frappe.cache.get_keys(""))
+		self.cache_memory_usage = frappe.cache.execute_command("INFO", "MEMORY").get("used_memory_human")
+
+	@health_check("Storage")
+	def fetch_storage_details(self):
+		from frappe.desk.page.backups.backups import get_context
+
+		self.backups_size = get_directory_size("private", "backups") / (1024 * 1024)
+		self.private_files_size = get_directory_size("private", "files") / (1024 * 1024)
+		self.public_files_size = get_directory_size("public", "files") / (1024 * 1024)
+		self.onsite_backups = len(get_context({}).get("files", []))
+
+	@health_check("Users")
+	def fetch_user_stats(self):
+		threshold = add_to_date(None, days=-30, as_datetime=True)
+		self.total_users = frappe.db.count("User", {"enabled": 1})
+		self.new_users = frappe.db.count("User", {"enabled": 1, "creation": (">", threshold)})
+		self.failed_logins = frappe.db.count(
+			"Activity Log",
+			{
+				"operation": "login",
+				"status": "Failed",
+				"creation": (">", threshold),
+				"modified": (">", threshold),
+			},
+		)
+		self.active_sessions = frappe.db.count("Sessions")
+		self.last_10_active_users = "\n".join(
+			frappe.get_all(
+				"User",
+				{"enabled": 1},
+				order_by="last_active desc",
+				limit=10,
+				pluck="name",
+			)
+		)
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_stats(**kwargs):
+		raise NotImplementedError
+
+
+@frappe.whitelist()
+def get_job_status(job_id: str | None = None):
+	frappe.only_for("System Manager")
+	try:
+		return frappe.get_doc("RQ Job", job_id).status
+	except Exception:
+		frappe.clear_messages()
+
+
+@redis_cache(ttl=5 * 60)
+def get_directory_size(*path):
+	return _get_directory_size(*path)
+
+
+def _get_directory_size(*path):
+	folder = os.path.abspath(frappe.get_site_path(*path))
+	# Copied as is from agent
+	total_size = os.path.getsize(folder)
+	for item in os.listdir(folder):
+		itempath = os.path.join(folder, item)
+
+		if not os.path.islink(itempath):
+			if os.path.isfile(itempath):
+				total_size += os.path.getsize(itempath)
+			elif os.path.isdir(itempath):
+				total_size += _get_directory_size(itempath)
+	return total_size

frappe/desk/doctype/system_health_report/test_system_health_report.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2024, Frappe Technologies and Contributors
+# See license.txt
+
+import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestSystemHealthReport(FrappeTestCase):
+	def test_it_works(self):
+		frappe.get_doc("System Health Report")

frappe/desk/doctype/system_health_report_errors/system_health_report_errors.json

@@ -0,0 +1,39 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-19 17:02:48.566902",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "title",
+  "occurrences"
+ ],
+ "fields": [
+  {
+   "fieldname": "title",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Title"
+  },
+  {
+   "fieldname": "occurrences",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Occurrences"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2024-04-19 17:10:43.199907",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report Errors",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report_errors/system_health_report_errors.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class SystemHealthReportErrors(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		occurrences: DF.Int
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		title: DF.Data | None
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		raise NotImplementedError
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		pass
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(**kwargs):
+		pass

frappe/desk/doctype/system_health_report_failing_jobs/system_health_report_failing_jobs.json

@@ -0,0 +1,40 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-22 11:45:32.923379",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "scheduled_job_type",
+  "failure_rate"
+ ],
+ "fields": [
+  {
+   "fieldname": "scheduled_job_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Scheduled Job Type",
+   "options": "Scheduled Job Type"
+  },
+  {
+   "fieldname": "failure_rate",
+   "fieldtype": "Percent",
+   "in_list_view": 1,
+   "label": "Failure Rate"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2024-04-22 11:46:53.574720",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report Failing Jobs",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report_failing_jobs/system_health_report_failing_jobs.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class SystemHealthReportFailingJobs(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		failure_rate: DF.Percent
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		scheduled_job_type: DF.Link | None
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		raise NotImplementedError
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		pass
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(**kwargs):
+		pass

frappe/desk/doctype/system_health_report_queue/system_health_report_queue.json

@@ -0,0 +1,39 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-19 15:47:47.592170",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "queue",
+  "pending_jobs"
+ ],
+ "fields": [
+  {
+   "fieldname": "queue",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Queue"
+  },
+  {
+   "fieldname": "pending_jobs",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Pending Jobs"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2024-04-19 15:47:47.592170",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report Queue",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report_queue/system_health_report_queue.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class SystemHealthReportQueue(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		pending_jobs: DF.Int
+		queue: DF.Data | None
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		raise NotImplementedError
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		pass
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(**kwargs):
+		pass

frappe/desk/doctype/system_health_report_tables/system_health_report_tables.json

@@ -0,0 +1,39 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-19 15:46:57.993123",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "table",
+  "size"
+ ],
+ "fields": [
+  {
+   "fieldname": "table",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Table"
+  },
+  {
+   "fieldname": "size",
+   "fieldtype": "Float",
+   "in_list_view": 1,
+   "label": "Size (MB)"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2024-04-19 15:46:57.993123",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report Tables",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report_tables/system_health_report_tables.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class SystemHealthReportTables(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		size: DF.Float
+		table: DF.Data | None
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		raise NotImplementedError
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		pass
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(**kwargs):
+		pass

frappe/desk/doctype/system_health_report_workers/system_health_report_workers.json

@@ -0,0 +1,52 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-04-19 15:44:52.298443",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "queues",
+  "count",
+  "utilization",
+  "failed_jobs"
+ ],
+ "fields": [
+  {
+   "fieldname": "queues",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Queues"
+  },
+  {
+   "fieldname": "count",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Count"
+  },
+  {
+   "fieldname": "utilization",
+   "fieldtype": "Percent",
+   "in_list_view": 1,
+   "label": "Utilization"
+  },
+  {
+   "fieldname": "failed_jobs",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Failed Jobs"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2024-04-19 15:44:52.298443",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "System Health Report Workers",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "creation",
+ "sort_order": "DESC",
+ "states": []
+}

frappe/desk/doctype/system_health_report_workers/system_health_report_workers.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2024, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class SystemHealthReportWorkers(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		count: DF.Int
+		failed_jobs: DF.Int
+		parent: DF.Data
+		parentfield: DF.Data
+		parenttype: DF.Data
+		queues: DF.Data | None
+		utilization: DF.Percent
+	# end: auto-generated types
+
+	def db_insert(self, *args, **kwargs):
+		raise NotImplementedError
+
+	def load_from_db(self):
+		raise NotImplementedError
+
+	def db_update(self):
+		raise NotImplementedError
+
+	def delete(self):
+		raise NotImplementedError
+
+	@staticmethod
+	def get_list(filters=None, page_length=20, **kwargs):
+		pass
+
+	@staticmethod
+	def get_count(filters=None, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(**kwargs):
+		pass

frappe/desk/page/backups/backups.py

@@ -29,9 +29,6 @@ def get_context(context):
 	files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))]
 	backup_limit = get_scheduled_backup_limit()
 
-	if len(files) > backup_limit:
-		cleanup_old_backups(path, files, backup_limit)
-
 	files = [
 		(
 			"/backups/" + _file,

frappe/email/doctype/notification/notification.js

@@ -97,8 +97,8 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 <h4>Details</h4>
 
 <ul>
-<li>Customer: {{ doc.customer }}
-<li>Amount: {{ doc.grand_total }}
+<li>Customer: {{ doc.customer }}</li>
+<li>Amount: {{ doc.grand_total }}</li>
 </ul>
 </pre>
 			`;

frappe/email/receive.py

@@ -44,6 +44,10 @@ poplib._MAXLINE = 1_00_000
 THREAD_ID_PATTERN = re.compile(r"(?<=\[)[\w/-]+")
 WORDS_PATTERN = re.compile(r"\w+")
 
+ALTERNATE_CHARSET_MAP = {
+	"windows-874": "cp874",
+}
+
 
 class EmailSizeExceededError(frappe.ValidationError):
 	pass
@@ -406,9 +410,9 @@ class Email:
 		_subject = decode_header(self.mail.get("Subject", "No Subject"))
 		self.subject = _subject[0][0] or ""
 
-		if _subject[0][1]:
+		if charset := _subject[0][1]:
 			# Encoding is known by decode_header (might also be unknown-8bit)
-			self.subject = safe_decode(self.subject, _subject[0][1])
+			self.subject = safe_decode(self.subject, charset, ALTERNATE_CHARSET_MAP)
 
 		if isinstance(self.subject, bytes):
 			# Fall back to utf-8 if the charset is unknown or decoding fails
@@ -502,11 +506,15 @@ class Email:
 
 	def get_payload(self, part):
 		charset = self.get_charset(part)
-
 		try:
 			return str(part.get_payload(decode=True), str(charset), "ignore")
 		except LookupError:
-			return part.get_payload()
+			try:
+				return str(
+					part.get_payload(decode=True), ALTERNATE_CHARSET_MAP.get(charset, "utf-8"), "ignore"
+				)
+			except Exception:
+				return part.get_payload()
 
 	def get_attachment(self, part):
 		# charset = self.get_charset(part)

frappe/handler.py

@@ -124,45 +124,6 @@ def web_logout():
 	)
 
 
-@frappe.whitelist()
-def uploadfile():
-	ret = None
-	check_write_permission(frappe.form_dict.doctype, frappe.form_dict.docname)
-
-	try:
-		if frappe.form_dict.get("from_form"):
-			try:
-				ret = frappe.get_doc(
-					{
-						"doctype": "File",
-						"attached_to_name": frappe.form_dict.docname,
-						"attached_to_doctype": frappe.form_dict.doctype,
-						"attached_to_field": frappe.form_dict.docfield,
-						"file_url": frappe.form_dict.file_url,
-						"file_name": frappe.form_dict.filename,
-						"is_private": frappe.utils.cint(frappe.form_dict.is_private),
-						"content": frappe.form_dict.filedata,
-						"decode": True,
-					}
-				)
-				ret.save()
-			except frappe.DuplicateEntryError:
-				# ignore pass
-				ret = None
-				frappe.db.rollback()
-		else:
-			if frappe.form_dict.get("method"):
-				method = frappe.get_attr(frappe.form_dict.method)
-				is_whitelisted(method)
-				ret = method()
-	except Exception:
-		frappe.errprint(frappe.utils.get_traceback())
-		frappe.response["http_status_code"] = 500
-		ret = None
-
-	return ret
-
-
 @frappe.whitelist(allow_guest=True)
 def upload_file():
 	user = None
@@ -252,7 +213,8 @@ def check_write_permission(doctype: str | None = None, name: str | None = None):
 			doc.check_permission("write")
 		except frappe.DoesNotExistError:
 			# doc has not been inserted yet, name is set to "new-some-doctype"
-			check_doctype = True
+			# If doc inserts fine then only this attachment will be linked see file/utils.py:relink_mismatched_files
+			return
 
 	if check_doctype:
 		frappe.has_permission(doctype, "write", throw=True)

frappe/hooks.py

@@ -524,6 +524,12 @@ standard_help_items = [
 		"action": "frappe.ui.toolbar.show_shortcuts(event)",
 		"is_standard": 1,
 	},
+	{
+		"item_label": "System Health",
+		"item_type": "Route",
+		"route": "/app/system-health-report",
+		"is_standard": 1,
+	},
 	{
 		"item_label": "Frappe Support",
 		"item_type": "Route",

frappe/integrations/doctype/webhook/test_webhook.py

@@ -77,6 +77,8 @@ class TestWebhook(FrappeTestCase):
 
 	def setUp(self):
 		# retrieve or create a User webhook for `after_insert`
+		self.responses = responses.RequestsMock()
+		self.responses.start()
 		webhook_fields = {
 			"webhook_doctype": "User",
 			"webhook_docevent": "after_insert",
@@ -101,9 +103,6 @@ class TestWebhook(FrappeTestCase):
 		self.test_user.first_name = "user1"
 		self.test_user.send_welcome_email = False
 
-		self.responses = responses.RequestsMock()
-		self.responses.start()
-
 	def tearDown(self) -> None:
 		self.user.delete()
 		self.test_user.delete()

frappe/integrations/doctype/webhook/webhook.py

@@ -213,7 +213,7 @@ def log_request(
 			"url": url,
 			"headers": frappe.as_json(headers) if headers else None,
 			"data": frappe.as_json(data) if data else None,
-			"response": res and res.text,
+			"response": res.text if res is not None else None,
 			"error": frappe.get_traceback(),
 		}
 	)

frappe/model/rename_doc.py

@@ -463,7 +463,12 @@ def get_link_fields(doctype: str) -> list[dict]:
 			.inner_join(dt)
 			.on(df.parent == dt.name)
 			.select(df.parent, df.fieldname, dt.issingle.as_("issingle"))
-			.where((df.options == doctype) & (df.fieldtype == "Link") & (dt.is_virtual == 0))
+			.where(
+				(df.options == doctype)
+				& (df.fieldtype == "Link")
+				& (df.is_virtual == 0)
+				& (dt.is_virtual == 0)
+			)
 			.run(as_dict=True)
 		)
 

frappe/oauth.py

@@ -11,7 +11,7 @@ from oauthlib.openid import RequestValidator
 
 import frappe
 from frappe.auth import LoginManager
-from frappe.utils.data import get_system_timezone, now_datetime
+from frappe.utils.data import cstr, get_system_timezone, now_datetime
 
 
 class OAuthWebRequestValidator(RequestValidator):
@@ -29,8 +29,10 @@ class OAuthWebRequestValidator(RequestValidator):
 		# Is the client allowed to use the supplied redirect_uri? i.e. has
 		# the client previously registered this EXACT redirect uri.
 
-		redirect_uris = frappe.db.get_value("OAuth Client", client_id, "redirect_uris").split(
-			get_url_delimiter()
+		redirect_uris = (
+			cstr(frappe.db.get_value("OAuth Client", client_id, "redirect_uris"))
+			.strip()
+			.split(get_url_delimiter())
 		)
 
 		if redirect_uri in redirect_uris:

frappe/permissions.py

@@ -451,7 +451,7 @@ def has_controller_permissions(doc, ptype, user=None, debug=False) -> bool:
 
 
 def get_doctypes_with_read():
-	return list({cstr(p.parent) for p in get_valid_perms() if p.parent})
+	return list({cstr(p.parent) for p in get_valid_perms() if p.parent and p.read})
 
 
 def get_valid_perms(doctype=None, user=None):

frappe/public/icons/espresso/icons.svg

@@ -346,6 +346,11 @@
         <path fill="var(--icon-stroke)" fill-rule="evenodd" d="M6.55 2a1 1 0 0 0-1 1v6.9a1 1 0 0 0 1 1h5.25a1 1 0 0 0 1-1.001V6.126H9.675a1 1 0 0 1-1-1V2.5a.5.5 0 0 0-.5-.5H6.55Zm7.25 4.126v-.653a2 2 0 0 0-.671-1.494l-2.783-2.474A2 2 0 0 0 9.017 1H6.55a2 2 0 0 0-2 1.953.505.505 0 0 0-.05-.003H4a2 2 0 0 0-2 2v7.5a2 2 0 0 0 2 2h5.453a2.057 2.057 0 0 0 2.047-2.048V11.9h.3a2 2 0 0 0 2-2.001V6.126ZM10.5 11.9H6.55a2 2 0 0 1-2-2V3.948a.506.506 0 0 1-.05.002H4a1 1 0 0 0-1 1v7.5a1 1 0 0 0 1 1h5.453c.571 0 1.047-.476 1.047-1.047V11.9Zm1.964-7.174a1 1 0 0 1 .273.4H9.675V2.5c0-.094-.009-.185-.025-.274a1 1 0 0 1 .031.027l2.783 2.473Z" class="Union" clip-rule="evenodd"/>
       </g>
     </symbol>
+    <symbol id="es-line-copy-light" fill="none" viewBox="0 0 16 16">
+      <g class="es-line-copy-light">
+        <path fill="var(--text-light)" fill-rule="evenodd" d="M6.55 2a1 1 0 0 0-1 1v6.9a1 1 0 0 0 1 1h5.25a1 1 0 0 0 1-1.001V6.126H9.675a1 1 0 0 1-1-1V2.5a.5.5 0 0 0-.5-.5H6.55Zm7.25 4.126v-.653a2 2 0 0 0-.671-1.494l-2.783-2.474A2 2 0 0 0 9.017 1H6.55a2 2 0 0 0-2 1.953.505.505 0 0 0-.05-.003H4a2 2 0 0 0-2 2v7.5a2 2 0 0 0 2 2h5.453a2.057 2.057 0 0 0 2.047-2.048V11.9h.3a2 2 0 0 0 2-2.001V6.126ZM10.5 11.9H6.55a2 2 0 0 1-2-2V3.948a.506.506 0 0 1-.05.002H4a1 1 0 0 0-1 1v7.5a1 1 0 0 0 1 1h5.453c.571 0 1.047-.476 1.047-1.047V11.9Zm1.964-7.174a1 1 0 0 1 .273.4H9.675V2.5c0-.094-.009-.185-.025-.274a1 1 0 0 1 .031.027l2.783 2.473Z" class="Union" clip-rule="evenodd"/>
+      </g>
+    </symbol>
     <symbol id="es-line-edit" fill="none" viewBox="0 0 16 16">
       <g class="es-line-edit">
         <path fill="var(--icon-stroke)" fill-rule="evenodd" d="M2.5 4.5a2 2 0 0 1 2-2h4a.5.5 0 0 0 0-1h-4a3 3 0 0 0-3 3v7a3 3 0 0 0 3 3h7a3 3 0 0 0 3-3v-4a.5.5 0 0 0-1 0v4a2 2 0 0 1-2 2h-7a2 2 0 0 1-2-2v-7Zm11.626-1.916a.5.5 0 0 0-.708-.707L6.686 8.61a.5.5 0 0 0 .707.707l6.733-6.733Z" class="Union" clip-rule="evenodd"/>

frappe/public/js/frappe/form/controls/code.js

@@ -4,6 +4,39 @@ frappe.ui.form.ControlCode = class ControlCode extends frappe.ui.form.ControlTex
 		this.load_lib().then(() => this.make_ace_editor());
 	}
 
+	make_wrapper() {
+		super.make_wrapper();
+		this.set_copy_button();
+	}
+
+	set_copy_button() {
+		if (!this.frm?.doc) {
+			return;
+		}
+
+		const codeField = this.df.fieldtype === "Code";
+		if ((codeField && this.df.read_only === 1) || (codeField && this.frm.doc.docstatus > 0)) {
+			this.button = $(
+				`<button
+					class="btn icon-btn"
+					style="position: absolute; top: 32px; right: 5px;"
+					onmouseover="this.classList.add('btn-default')"
+					onmouseout="this.classList.remove('btn-default')"
+				>
+					<svg class="es-icon es-line  icon-sm" style="" aria-hidden="true">
+						<use class="" href="#es-line-copy-light"></use>
+					</svg>
+				</button>`
+			);
+			this.button.on("click", () => {
+				frappe.utils.copy_to_clipboard(
+					frappe.model.get_value(this.doctype, this.docname, this.df.fieldname)
+				);
+			});
+			this.button.appendTo(this.$wrapper);
+		}
+	}
+
 	make_ace_editor() {
 		if (this.editor) return;
 		this.ace_editor_target = $('<div class="ace-editor-target"></div>').appendTo(

frappe/public/js/frappe/form/grid.js

@@ -795,7 +795,7 @@ export default class Grid {
 	}
 
 	set_value(fieldname, value, doc) {
-		if (this.display_status !== "None" && doc.name && this.grid_rows_by_docname[doc.name]) {
+		if (this.display_status !== "None" && doc?.name && this.grid_rows_by_docname[doc.name]) {
 			this.grid_rows_by_docname[doc.name].refresh_field(fieldname, value);
 		}
 	}

frappe/public/js/frappe/views/file/file_view.js

@@ -191,6 +191,8 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 	prepare_datum(d) {
 		let icon_class = "";
 		let type = "";
+		let title;
+
 		if (d.is_folder) {
 			icon_class = "folder-normal";
 			type = "folder";
@@ -202,7 +204,12 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 			type = "file";
 		}
 
-		let title = d.file_name || d.file_url;
+		if (type === "folder") {
+			title = this.get_folder_title(d.file_name);
+		} else {
+			title = d.file_name || d.file_url;
+		}
+
 		title = title.slice(0, 60);
 		d._title = title;
 		d.icon_class = icon_class;
@@ -216,6 +223,16 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 		return d;
 	}
 
+	get_folder_title(folder_name) {
+		// "Home" and "Attachments" are default folders that are always created in english.
+		// So we can and should translate them to the user's language.
+		if (["Home", "Attachments"].includes(folder_name)) {
+			return __(folder_name);
+		} else {
+			return folder_name;
+		}
+	}
+
 	before_render() {
 		super.before_render();
 		frappe.model.user_settings.save("File", "grid_view", frappe.views.FileView.grid_view);
@@ -290,8 +307,10 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 
 		return folders
 			.map((folder, i) => {
+				const title = this.get_folder_title(folder);
+
 				if (i === folders.length - 1) {
-					return `<span>${folder}</span>`;
+					return `<span>${title}</span>`;
 				}
 				const route = folders.reduce((acc, curr, j) => {
 					if (j <= i) {
@@ -300,7 +319,7 @@ frappe.views.FileView = class FileView extends frappe.views.ListView {
 					return acc;
 				}, "/app/file/view");
 
-				return `<a href="${route}">${folder}</a>`;
+				return `<a href="${route}">${title}</a>`;
 			})
 			.join("&nbsp;/&nbsp;");
 	}

frappe/public/js/frappe/views/image/photoswipe_dom.html

@@ -13,7 +13,7 @@
 
 		<!-- Container that holds slides.
 			PhotoSwipe keeps only 3 of them in the DOM to save memory.
-			Don't modify these 3 pswp__item elements, data is added later on. -->
+			Do not modify these 3 pswp__item elements, data is added later on. -->
 		<div class="pswp__container">
 			<div class="pswp__item"></div>
 			<div class="pswp__item"></div>
@@ -70,4 +70,4 @@
 
 	</div>
 
-</div>
+</div>

frappe/public/js/frappe/views/reports/query_report.js

@@ -68,21 +68,22 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 	}
 
 	get_url_with_filters() {
-		const query_params = Object.entries(this.get_filter_values())
-			.map(([field, value], _idx) => {
+		let query_params = new URLSearchParams();
+		if (this.prepared_report_name) {
+			query_params.append("prepared_report_name", this.prepared_report_name);
+		} else {
+			Object.entries(this.get_filter_values()).map(([field, value], _idx) => {
 				// multiselects
 				if (Array.isArray(value)) {
 					if (!value.length) return "";
 					value = JSON.stringify(value);
 				}
-				return `${field}=${encodeURIComponent(value)}`;
-			})
-			.filter(Boolean)
-			.join("&");
-
+				query_params.append(field, value);
+			});
+		}
 		let full_url = window.location.href.replace(window.location.search, "");
-		if (query_params) {
-			full_url += "?" + query_params;
+		if (query_params.toString()) {
+			full_url += "?" + query_params.toString();
 		}
 		return full_url;
 	}
@@ -394,6 +395,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 	}
 
 	refresh_report(route_options) {
+		this.prepared_report_name = null; // this should be set only if prepared report is EXPLICITLY requested
 		this.toggle_message(true);
 		this.toggle_report(false);
 
@@ -596,6 +598,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			const fields = Object.keys(route_options);
 
 			const filters_to_set = this.filters.filter((f) => fields.includes(f.df.fieldname));
+			this.prepared_report_name = route_options.prepared_report_name;
 
 			const promises = filters_to_set.map((f) => {
 				return () => {
@@ -647,10 +650,8 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			this.last_ajax.abort();
 		}
 
-		const query_params = this.get_query_params();
-
-		if (query_params.prepared_report_name) {
-			filters.prepared_report_name = query_params.prepared_report_name;
+		if (this.prepared_report_name) {
+			filters.prepared_report_name = this.prepared_report_name;
 		}
 
 		return new Promise((resolve) => {
@@ -686,7 +687,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 					this.prepared_report_document = data.doc;
 					// If query_string contains prepared_report_name then set filters
 					// to match the mentioned prepared report doc and disable editing
-					if (query_params.prepared_report_name) {
+					if (this.prepared_report_name) {
 						this.prepared_report_action = "Edit";
 						const filters_from_report = JSON.parse(data.doc.filters);
 						Object.values(this.filters).forEach(function (field) {
@@ -958,7 +959,7 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 		let data = this.data;
 		let columns = this.columns.filter((col) => !col.hidden);
 
-		if (data.length > 100000) {
+		if (data.length > 1000000) {
 			let msg = __(
 				"This report contains {0} rows and is too big to display in browser, you can {1} this report instead.",
 				[cstr(format_number(data.length, null, 0)).bold(), __("export").bold()]
@@ -1522,12 +1523,6 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 				this.make_access_log("Export", file_format);
 
 				let filters = this.get_filter_values(true);
-				if (frappe.urllib.get_dict("prepared_report_name")) {
-					filters = Object.assign(
-						frappe.urllib.get_dict("prepared_report_name"),
-						filters
-					);
-				}
 				let boolean_labels = { 1: __("Yes"), 0: __("No") };
 				let applied_filters = Object.fromEntries(
 					Object.entries(filters).map(([key, value]) => [
@@ -1537,6 +1532,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 							: value,
 					])
 				);
+				if (this.prepared_report_name) {
+					filters.prepared_report_name = this.prepared_report_name;
+				}
 
 				const visible_idx = this.datatable?.bodyRenderer.visibleRowIndices || [];
 				if (visible_idx.length + 1 === this.data?.length) {

frappe/public/js/frappe/views/reports/report_view.js

@@ -1553,12 +1553,22 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView {
 					const selected_items = this.get_checked_items(true);
 
 					let extra_fields = null;
-					if (this.total_count > (this.count_without_children || args.page_length)) {
+					if (this.list_view_settings.disable_count) {
 						extra_fields = [
 							{
 								fieldtype: "Check",
 								fieldname: "export_all_rows",
-								label: __("Export All {0} rows?", [`<b>${this.total_count}</b>`]),
+								label: __("Export all matching rows?"),
+							},
+						];
+					} else if (
+						this.total_count > (this.count_without_children || args.page_length)
+					) {
+						extra_fields = [
+							{
+								fieldtype: "Check",
+								fieldname: "export_all_rows",
+								label: __("Export all {0} rows?", [`<b>${this.total_count}</b>`]),
 							},
 						];
 					}

frappe/public/scss/common/datepicker.scss

@@ -90,7 +90,7 @@
 	}
 
 	&--time-row {
-		background-image: linear-gradient(to right, var(--gray-900), var(--gray-900));
+		background-image: linear-gradient(to right, var(--gray-600), var(--gray-600));
 		background-repeat: no-repeat;
 		background-size: 100% 1px;
 		background-position: left 50%;

frappe/public/scss/desk/desktop.scss

@@ -636,6 +636,16 @@ body {
 			@include get_textstyle("lg", "semibold");
 			color: var(--text-color) !important;
 		}
+
+		@include media-breakpoint-down(md) {
+			padding: 1px 8px;
+
+			.link-item {
+				&:first-child {
+					margin-top: 5px;
+				}
+			}
+		}
 	}
 
 	&.number-widget-box {
@@ -1264,6 +1274,10 @@ body {
 					height: 100%;
 					padding: 7px;
 
+					@include media-breakpoint-down(md) {
+						padding: 7px 3px;
+					}
+
 					& > div {
 						height: 100%;
 					}
@@ -1405,6 +1419,13 @@ body {
 						&:focus {
 							outline: none;
 						}
+
+						@include media-breakpoint-down(md) {
+							&.spacer {
+								height: 0 !important;
+								padding: 0 7px;
+							}
+						}
 					}
 				}
 			}
@@ -1485,7 +1506,7 @@ body {
 				max-width: 930px;
 			}
 		}
-		@media (max-width: 995px) {
+		@include media-breakpoint-down(md) {
 			.ce-toolbar__content {
 				max-width: 760px;
 			}

frappe/public/scss/desk/form.scss

@@ -215,7 +215,7 @@
 			background-color: var(--gray-100);
 			font-size: var(--text-xs);
 			padding: 0 var(--padding-sm);
-			color: var(--indicator-red);
+			color: var(--gray-700);
 			border-radius: var(--border-radius);
 			cursor: pointer;
 		}

frappe/tests/test_permissions.py

@@ -5,8 +5,9 @@
 import frappe
 import frappe.defaults
 import frappe.model.meta
+from frappe.core.doctype.doctype.test_doctype import new_doctype
 from frappe.core.doctype.user_permission.user_permission import clear_user_permissions
-from frappe.core.page.permission_manager.permission_manager import reset, update
+from frappe.core.page.permission_manager.permission_manager import add, remove, reset, update
 from frappe.desk.form.load import getdoc
 from frappe.permissions import (
 	ALL_USER_ROLE,
@@ -17,6 +18,7 @@ from frappe.permissions import (
 	add_user_permission,
 	clear_user_permissions_for_doctype,
 	get_doc_permissions,
+	get_doctypes_with_read,
 	remove_user_permission,
 	update_permission_property,
 )
@@ -736,3 +738,29 @@ class TestPermissions(FrappeTestCase):
 		)
 		frappe.set_user(system_user)
 		assertHasRole(GUEST_ROLE, ALL_USER_ROLE, SYSTEM_USER_ROLE)
+
+	def test_get_doctypes_with_read(self):
+		with self.set_user("Administrator"):
+			doctype = new_doctype(permissions=[{"select": 1, "role": "_Test Role", "read": 0}]).insert().name
+
+		with self.set_user("test@example.com"):
+			self.assertNotIn(doctype, get_doctypes_with_read())
+
+	def test_overrides_work_as_expected(self):
+		"""custom docperms should completely override standard ones"""
+		standard_role = "Desk User"
+		custom_role = frappe.new_doc("Role", role_name=frappe.generate_hash()).insert().name
+		with self.set_user("Administrator"):
+			doctype = new_doctype(permissions=[{"role": standard_role, "read": 1}]).insert().name
+
+		with self.set_user("test@example.com"):
+			self.assertIn(doctype, get_doctypes_with_read())
+
+		with self.set_user("Administrator"):
+			# Allow perm to some other role and remove standard role
+			add(doctype, custom_role, 0)
+			remove(doctype, standard_role, 0)
+
+		with self.set_user("test@example.com"):
+			# No one has this role, so user shouldn't have permission.
+			self.assertNotIn(doctype, get_doctypes_with_read())

frappe/utils/__init__.py

@@ -229,7 +229,7 @@ def validate_url(
 	        valid_schemes: if provided checks the given URL's scheme against this
 	"""
 	url = urlparse(txt)
-	is_valid = bool(url.netloc)
+	is_valid = bool(url.netloc) or (txt and txt.startswith("/"))
 
 	# Handle scheme validation
 	if isinstance(valid_schemes, str):

frappe/utils/pdf.py

@@ -276,7 +276,7 @@ def _get_base64_image(src):
 		path = parsed_url.path
 		query = parse_qs(parsed_url.query)
 		mime_type = mimetypes.guess_type(path)[0]
-		if not mime_type.startswith("image/"):
+		if mime_type is None or not mime_type.startswith("image/"):
 			return
 		filename = query.get("fid") and query["fid"][0] or None
 		file = find_file_by_url(path, name=filename)

frappe/utils/sentry.py

@@ -125,8 +125,8 @@ def capture_exception(message: str | None = None) -> None:
 		if client := hub.client:
 			exc_info = sys.exc_info()
 			if any(exc_info):
-				# Don't report validation errors
-				if isinstance(exc_info[1], frappe.ValidationError):
+				# Don't report errors which we can't "fix" in code
+				if isinstance(exc_info[1], frappe.ValidationError | frappe.PermissionError):
 					return
 
 				event, hint = event_from_exception(

frappe/website/doctype/blog_post/ui_test_blog_post.js

@@ -1,36 +0,0 @@
-context("Blog Post", () => {
-	before(() => {
-		cy.login();
-		cy.visit("/app");
-	});
-
-	it("Blog Category dropdown works as expected", () => {
-		cy.create_records([
-			{
-				doctype: "Blog Category",
-				title: "Category 1",
-				published: 1,
-			},
-			{
-				doctype: "Blogger",
-				short_name: "John",
-				full_name: "John Doe",
-			},
-			{
-				doctype: "Blog Post",
-				title: "Test Blog Post",
-				content: "Test Blog Post Content",
-				blog_category: "category-1",
-				blogger: "John",
-				published: 1,
-			},
-		]);
-		cy.set_value("Blog Settings", "Blog Settings", { browse_by_category: 1 });
-		cy.visit("/blog");
-		cy.findByLabelText("Browse by category").select("Category 1");
-		cy.location("pathname").should("eq", "/blog/category-1");
-		cy.set_value("Blog Settings", "Blog Settings", { browse_by_category: 0 });
-		cy.visit("/blog");
-		cy.findByLabelText("Browse by category").should("not.exist");
-	});
-});

frappe/website/doctype/web_form/web_form.py

@@ -664,7 +664,7 @@ def get_link_options(web_form_name, doctype, allow_read_on_all_link_options=Fals
 		)
 
 	link_options, filters = [], {}
-	if not allow_read_on_all_link_options:
+	if web_form.login_required and not allow_read_on_all_link_options:
 		filters = {"owner": frappe.session.user}
 
 	fields = ["name as value"]

frappe/website/path_resolver.py

@@ -1,6 +1,7 @@
 import re
 
 import click
+import werkzeug.routing.exceptions
 from werkzeug.routing import Rule
 
 import frappe
@@ -42,7 +43,11 @@ class PathResolver:
 			for handler in frappe.get_hooks("website_path_resolver"):
 				endpoint = frappe.get_attr(handler)(self.path)
 		else:
-			endpoint = resolve_path(self.path)
+			try:
+				endpoint = resolve_path(self.path)
+			except werkzeug.routing.exceptions.RequestRedirect as e:
+				frappe.flags.redirect_location = e.new_url
+				return frappe.flags.redirect_location, RedirectPage(e.new_url, e.code)
 
 		# WARN: Hardcoded for better performance
 		if endpoint == "app":

frappe/website/router.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import io
 import os
 import re
 
@@ -71,12 +70,10 @@ def evaluate_dynamic_routes(rules, path):
 		urls = route_map.bind_to_environ(frappe.local.request.environ)
 		try:
 			endpoint, args = urls.match("/" + path)
-			path = endpoint
 			if args:
 				# don't cache when there's a query string!
 				frappe.local.no_cache = 1
 				frappe.local.form_dict.update(args)
-
 		except NotFound:
 			pass
 
@@ -110,10 +107,6 @@ def get_pages_from_path(start, app, app_path):
 	start_path = os.path.join(app_path, start)
 	if os.path.exists(start_path):
 		for basepath, folders, files in os.walk(start_path):  # noqa: B007
-			# add missing __init__.py
-			if "__init__.py" not in files and frappe.conf.get("developer_mode"):
-				open(os.path.join(basepath, "__init__.py"), "a").close()
-
 			for fname in files:
 				fname = frappe.utils.cstr(fname)
 				if "." not in fname:
@@ -128,7 +121,6 @@ def get_pages_from_path(start, app, app_path):
 						os.path.join(basepath, fname), app, start, basepath, app_path, fname
 					)
 					pages[page_info.route] = page_info
-					# print frappe.as_json(pages[-1])
 
 	return pages
 

frappe/www/attribution.html

@@ -24,6 +24,7 @@
             <th>{{ _("Description") }}</th>
             <td>{{ app_info.description }}</td>
         </tr>
+        {% if app_info["dependencies"] %}
         <tr>
             <th>{{ _("Dependencies") }}</th>
             <td>
@@ -50,6 +51,8 @@
                     </tbody>
                 </table>
             </td>
+        </tr>
+        {% endif %}
     </table>
 </section>
 {% endfor %}
@@ -65,13 +68,13 @@
             var author_cell = license_cell.nextElementSibling;
             if (type_cell.innerText === "JavaScript") {
                 get_info_from_npm(name).then((info) => {
-                    license_cell.innerText = info.license;
+                    license_cell.innerText = info.license?.slice(0, 50);
                     author_cell.innerText = info.author;
                 });
             }
             else if (type_cell.innerText === "Python") {
                 get_info_from_pypi(name).then((info) => {
-                    license_cell.innerText = info.license;
+                    license_cell.innerText = info.license?.slice(0, 50);
                     author_cell.innerText = info.author;
                 });
             }

package.json

@@ -47,7 +47,7 @@
     "fast-deep-equal": "^2.0.1",
     "fast-glob": "^3.2.5",
     "frappe-charts": "2.0.0-rc22",
-    "frappe-datatable": "1.17.15",
+    "frappe-datatable": "1.17.16",
     "frappe-gantt": "^0.6.0",
     "highlight.js": "^10.4.1",
     "html5-qrcode": "^2.3.8",

realtime/handlers.js

@@ -12,7 +12,10 @@ function frappe_handlers(socket) {
 	socket.has_permission = (doctype, name) => {
 		return new Promise((resolve) => {
 			socket
-				.frappe_request("/api/method/frappe.realtime.has_permission", { doctype, name })
+				.frappe_request("/api/method/frappe.realtime.has_permission", {
+					doctype,
+					name: name || "",
+				})
 				.then((res) => res.json())
 				.then(({ message }) => {
 					if (message) {
@@ -23,6 +26,10 @@ function frappe_handlers(socket) {
 		});
 	};
 
+	socket.on("ping", () => {
+		socket.emit("pong");
+	});
+
 	socket.on("doctype_subscribe", function (doctype) {
 		socket.has_permission(doctype).then(() => {
 			socket.join(doctype_room(doctype));

yarn.lock

@@ -1424,10 +1424,10 @@ frappe-charts@2.0.0-rc22:
   resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc22.tgz#9a5a747febdc381a1d4d7af96e89cf519dfba8c0"
   integrity sha512-N7f/8979wJCKjusOinaUYfMxB80YnfuVLrSkjpj4LtyqS0BGS6SuJxUnb7Jl4RWUFEIs7zEhideIKnyLeFZF4Q==
 
-frappe-datatable@1.17.15:
-  version "1.17.15"
-  resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.15.tgz#c1665b1ca2c1446a3a239f2b2a985a6df0c9a789"
-  integrity sha512-/Zj5vwjUXX8UB/aC/oRvgZuSSj2saoKO1ux+w1MbUmhqK5B/sutct40Y+Nv/9+HAJswCb1UG6jNVa2IxUaGHQg==
+frappe-datatable@1.17.16:
+  version "1.17.16"
+  resolved "https://registry.yarnpkg.com/frappe-datatable/-/frappe-datatable-1.17.16.tgz#4e7bf3b50dad5bc048f95ccd7ca7da91e04844ab"
+  integrity sha512-BJgWFX8msHZcS1mw2xbuaY1YdH1dBXUIuREVmqH5z1p78GusPaDV8sbWskTS5yVBUklMrMq2VfBTUsJXjvl+wg==
   dependencies:
     hyperlist "^1.0.0-beta"
     lodash "^4.17.5"