Merge commit 'aeaf2b5' into 4.0-dev

administrator/components/com_admin/script.php

@@ -6254,11 +6254,13 @@ class JoomlaInstallerScript
 		// Set required conversion status
 		if ($db->hasUTF8mb4Support())
 		{
-			$converted = 2;
+			$convertedStep1 = 2;
+			$convertedStep2 = 4;
 		}
 		else
 		{
-			$converted = 1;
+			$convertedStep1 = 1;
+			$convertedStep2 = 3;
 		}
 
 		// Check conversion status in database
@@ -6284,73 +6286,122 @@ class JoomlaInstallerScript
 			return;
 		}
 
-		// Nothing to do, saved conversion status from DB is equal to required
-		if ($convertedDB == $converted)
+		// Nothing to do, saved conversion status from DB is equal to required final status
+		if ($convertedDB == $convertedStep2)
 		{
 			return;
 		}
 
-		// Step 1: Drop indexes later to be added again with column lengths limitations at step 2
-		$fileName1 = JPATH_ROOT . '/administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-01.sql';
+		$converted = $convertedDB;
+		$hasErrors = false;
 
-		if (is_file($fileName1))
+		// Steps 1 and 2: Convert core tables if necessary and not to be done at later steps
+		if ($convertedDB < $convertedStep1)
 		{
-			$fileContents1 = @file_get_contents($fileName1);
-			$queries1      = $db->splitSql($fileContents1);
+			// Step 1: Drop indexes later to be added again with column lengths limitations at step 2
+			$fileName1 = JPATH_ROOT . '/administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-01.sql';
 
-			if (!empty($queries1))
+			if (is_file($fileName1))
 			{
-				foreach ($queries1 as $query1)
+				$fileContents1 = @file_get_contents($fileName1);
+				$queries1      = $db->splitSql($fileContents1);
+
+				if (!empty($queries1))
 				{
-					try
+					foreach ($queries1 as $query1)
 					{
-						$db->setQuery($query1)->execute();
-					}
-					catch (Exception $e)
-					{
-						// If the query fails we will go on. It just means the index to be dropped does not exist.
+						try
+						{
+							$db->setQuery($query1)->execute();
+						}
+						catch (Exception $e)
+						{
+							// If the query fails we will go on. It just means the index to be dropped does not exist.
+						}
 					}
 				}
 			}
-		}
 
-		// Step 2: Perform the index modifications and conversions
-		$fileName2 = JPATH_ROOT . '/administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-02.sql';
+			// Step 2: Perform the index modifications and conversions
+			$fileName2 = JPATH_ROOT . '/administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-02.sql';
 
-		if (is_file($fileName2))
-		{
-			$fileContents2 = @file_get_contents($fileName2);
-			$queries2      = $db->splitSql($fileContents2);
-
-			if (!empty($queries2))
+			if (is_file($fileName2))
 			{
-				foreach ($queries2 as $query2)
-				{
-					try
-					{
-						$db->setQuery($db->convertUtf8mb4QueryToUtf8($query2))->execute();
-					}
-					catch (Exception $e)
-					{
-						$converted = 0;
+				$fileContents2 = @file_get_contents($fileName2);
+				$queries2      = $db->splitSql($fileContents2);
 
-						// Still render the error message from the Exception object
-						Factory::getApplication()->enqueueMessage($e->getMessage(), 'error');
+				if (!empty($queries2))
+				{
+					foreach ($queries2 as $query2)
+					{
+						try
+						{
+							$db->setQuery($db->convertUtf8mb4QueryToUtf8($query2))->execute();
+						}
+						catch (Exception $e)
+						{
+							$hasErrors = true;
+
+							// Still render the error message from the Exception object
+							JFactory::getApplication()->enqueueMessage($e->getMessage(), 'error');
+						}
 					}
 				}
 			}
+
+			if (!$hasErrors)
+			{
+				$converted = $convertedStep1;
+			}
 		}
 
-		if ($doDbFixMsg && $converted == 0)
+		// Step 3: Convert action logs and privacy suite tables if necessary and conversion hasn't failed before
+		if (!$hasErrors && $convertedDB < $convertedStep2)
+		{
+			$fileName3 = JPATH_ROOT . '/administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-03.sql';
+
+			if (is_file($fileName3))
+			{
+				$fileContents3 = @file_get_contents($fileName3);
+				$queries3      = $db->splitSql($fileContents3);
+
+				if (!empty($queries3))
+				{
+					foreach ($queries3 as $query3)
+					{
+						try
+						{
+							$db->setQuery($db->convertUtf8mb4QueryToUtf8($query3))->execute();
+						}
+						catch (Exception $e)
+						{
+							$hasErrors = true;
+
+							// Still render the error message from the Exception object
+							JFactory::getApplication()->enqueueMessage($e->getMessage(), 'error');
+						}
+					}
+				}
+			}
+
+			if (!$hasErrors)
+			{
+				$converted = $convertedStep2;
+			}
+		}
+
+		if ($doDbFixMsg && $hasErrors)
 		{
 			// Show an error message telling to check database problems
 			Factory::getApplication()->enqueueMessage(Text::_('JLIB_DATABASE_ERROR_DATABASE_UPGRADE_FAILED'), 'error');
 		}
 
-		// Set flag in database if the update is done.
-		$db->setQuery('UPDATE ' . $db->quoteName('#__utf8_conversion')
-			. ' SET ' . $db->quoteName('converted') . ' = ' . $converted . ';'
-		)->execute();
+		// Set flag in database if the conversion status has changed.
+		if ($converted != $convertedDB)
+		{
+			$db->setQuery('UPDATE ' . $db->quoteName('#__utf8_conversion')
+				. ' SET ' . $db->quoteName('converted') . ' = ' . $converted . ';')->execute();
+		}
 	}
 
 	/**

administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-02.sql

@@ -41,12 +41,12 @@ ALTER TABLE `#__newsfeeds` MODIFY `alias` varchar(400) NOT NULL DEFAULT '';
 ALTER TABLE `#__tags` MODIFY `path` varchar(400) NOT NULL DEFAULT '';
 ALTER TABLE `#__tags` MODIFY `alias` varchar(400) NOT NULL DEFAULT '';
 ALTER TABLE `#__ucm_content` MODIFY `core_type_alias` varchar(400) NOT NULL DEFAULT '' COMMENT 'FK to the content types table';
-ALTER TABLE `#__ucm_content` MODIFY `core_title` varchar(400) NOT NULL;
+ALTER TABLE `#__ucm_content` MODIFY `core_title` varchar(400) NOT NULL DEFAULT '';
 ALTER TABLE `#__ucm_content` MODIFY `core_alias` varchar(400) NOT NULL DEFAULT '';
 ALTER TABLE `#__users` MODIFY `name` varchar(400) NOT NULL DEFAULT '';
 
 --
--- Step 2.2: Convert all tables to utf8mb4 chracter set with utf8mb4_unicode_ci collation
+-- Step 2.2: Convert all tables to utf8mb4 character set with utf8mb4_unicode_ci collation
 -- except #__finder_xxx tables, those will have utf8mb4_general_ci collation.
 -- Note: The database driver for mysql will change utf8mb4 to utf8 if utf8mb4 is not supported
 --

administrator/components/com_admin/sql/others/mysql/utf8mb4-conversion-03.sql

@@ -0,0 +1,33 @@
+--
+-- Step 3 of the UTF-8 Multibyte (utf8mb4) conversion for MySQL
+--
+-- Convert the tables for action logs and the privacy suite which have been
+-- forgotten to be added to the utf8mb4 conversion before.
+--
+-- This file here will be processed with reporting exceptions, in opposite
+-- to the file for step 1.
+--
+
+--
+-- Step 3.1: Convert action logs and privacy suite tables to utf8mb4 character set with
+-- utf8mb4_unicode_ci collation
+-- Note: The database driver for mysql will change utf8mb4 to utf8 if utf8mb4 is not supported
+--
+
+ALTER TABLE `#__action_logs` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_logs_extensions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_logs_users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_log_config` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__privacy_consents` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__privacy_requests` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+--
+-- Step 3.2: Set default character set and collation for previously converted tables
+--
+
+ALTER TABLE `#__action_logs` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_logs_extensions` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_logs_users` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__action_log_config` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__privacy_consents` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+ALTER TABLE `#__privacy_requests` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

administrator/components/com_categories/src/Model/CategoryModel.php

@@ -752,6 +752,11 @@ class CategoryModel extends AdminModel
 
 		$this->setState($this->getName() . '.id', $table->id);
 
+		if (Factory::getApplication()->input->get('task') == 'editAssociations')
+		{
+			return $this->redirectToAssociations($data);
+		}
+
 		// Clear the cache
 		$this->cleanCache();
 

build/media_source/layouts/js/joomla/form/field/category-change.es6.js

@@ -29,6 +29,6 @@
       document.querySelector('input[name=task]').value = `${el.getAttribute('data-refresh-section')}.reload`;
     }
 
-    element.form.submit();
+    Joomla.submitform(`${el.getAttribute('data-refresh-section')}.reload`, element.form);
   };
 })(Joomla);

components/com_contact/src/Model/CategoryModel.php

@@ -501,7 +501,6 @@ class CategoryModel extends ListModel
 			$pk = (!empty($pk)) ? $pk : (int) $this->getState('category.id');
 
 			$table = Table::getInstance('Category');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_contact/src/Model/ContactModel.php

@@ -464,7 +464,6 @@ class ContactModel extends FormModel
 			$pk = $pk ?: (int) $this->getState('contact.id');
 
 			$table = $this->getTable('Contact');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_content/src/Model/ArticleModel.php

@@ -330,7 +330,6 @@ class ArticleModel extends ItemModel
 			$pk = (!empty($pk)) ? $pk : (int) $this->getState('article.id');
 
 			$table = Table::getInstance('Content', 'JTable');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_content/src/Model/CategoryModel.php

@@ -517,7 +517,6 @@ class CategoryModel extends ListModel
 			$pk = (!empty($pk)) ? $pk : (int) $this->getState('category.id');
 
 			$table = Table::getInstance('Category', 'JTable');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_newsfeeds/src/Model/CategoryModel.php

@@ -418,7 +418,6 @@ class CategoryModel extends ListModel
 		{
 			$pk    = (!empty($pk)) ? $pk : (int) $this->getState('category.id');
 			$table = Table::getInstance('Category', 'JTable');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_newsfeeds/src/Model/NewsfeedModel.php

@@ -227,7 +227,6 @@ class NewsfeedModel extends ItemModel
 			$pk = (!empty($pk)) ? $pk : (int) $this->getState('newsfeed.id');
 
 			$table = $this->getTable('Newsfeed', 'Administrator');
-			$table->load($pk);
 			$table->hit($pk);
 		}
 

components/com_tags/src/Model/TagModel.php

@@ -353,7 +353,6 @@ class TagModel extends ListModel
 
 			/** @var \Joomla\Component\Tags\Administrator\Table\Tag $table */
 			$table = $this->getTable();
-			$table->load($pk);
 			$table->hit($pk);
 
 			if (!$table->hasPrimaryKey())

installation/src/Model/DatabaseModel.php

@@ -570,7 +570,7 @@ class DatabaseModel extends BaseInstallationModel
 			$query = $db->getQuery(true);
 			$query->clear()
 				->update($db->quoteName('#__utf8_conversion'))
-				->set($db->quoteName('converted') . ' = ' . ($db->hasUTF8mb4Support() ? 2 : 1));
+				->set($db->quoteName('converted') . ' = ' . ($db->hasUTF8mb4Support() ? 4 : 3));
 			$db->setQuery($query);
 
 			try

libraries/src/Form/Field/FolderlistField.php

@@ -185,7 +185,7 @@ class FolderlistField extends ListField
 	{
 		$options = array();
 
-		$path = ltrim($this->directory, '/');
+		$path = $this->directory;
 
 		if (!is_dir($path))
 		{

libraries/src/Helper/ModuleHelper.php

@@ -711,7 +711,7 @@ abstract class ModuleHelper
 		for ($i = 0; $i < $total; $i++)
 		{
 			// Match the id of the module
-			if ($modules[$i]->id === $id)
+			if ((string) $modules[$i]->id === $id)
 			{
 				// Found it
 				return $modules[$i];

libraries/src/MVC/Controller/FormController.php

@@ -736,6 +736,26 @@ class FormController extends BaseController implements FormFactoryAwareInterface
 				}
 			}
 
+			/**
+			 * We need the filtered value of calendar fields because the UTC normalision is
+			 * done in the filter and on output. This would apply the Timezone offset on
+			 * reload. We set the calendar values we save to the processed date.
+			 */
+			$filteredData = $form->filter($data);
+
+			foreach ($form->getFieldset() as $field)
+			{
+				if ($field->type === 'Calendar')
+				{
+					$fieldName = $field->fieldname;
+
+					if (isset($filteredData[$fieldName]))
+					{
+						$data[$fieldName] = $filteredData[$fieldName];
+					}
+				}
+			}
+
 			// Save the data in the session.
 			$app->setUserState($context . '.data', $data);
 
@@ -913,6 +933,29 @@ class FormController extends BaseController implements FormFactoryAwareInterface
 			false
 		);
 
+		/* @var \JForm $form */
+		$form = $model->getForm($data, false);
+
+		/**
+		 * We need the filtered value of calendar fields because the UTC normalision is
+		 * done in the filter and on output. This would apply the Timezone offset on
+		 * reload. We set the calendar values we save to the processed date.
+		 */
+		$filteredData = $form->filter($data);
+
+		foreach ($form->getFieldset() as $field)
+		{
+			if ($field->type === 'Calendar')
+			{
+				$fieldName = $field->fieldname;
+
+				if (isset($filteredData[$fieldName]))
+				{
+					$data[$fieldName] = $filteredData[$fieldName];
+				}
+			}
+		}
+
 		// Save the data in the session.
 		$app->setUserState($this->option . '.edit.' . $this->context . '.data', $data);
 

libraries/src/MVC/Model/AdminModel.php

@@ -1667,7 +1667,7 @@ abstract class AdminModel extends FormModel
 	 * @return  boolean  True if successful, false otherwise.
 	 *
 	 * @throws \Exception
-	 * @since   __DEPLOY_VERSION__
+	 * @since   3.9.17
 	 */
 	protected function redirectToAssociations($data)
 	{

libraries/src/Schema/ChangeSet.php

@@ -99,12 +99,12 @@ class ChangeSet
 		// If on mysql, add a query at the end to check for utf8mb4 conversion status
 		if ($this->db->getServerType() === 'mysql')
 		{
-			// Let the update query be something harmless which should always succeed
+			// Let the update query do nothing when being executed
 			$tmpSchemaChangeItem = ChangeItem::getInstance(
 				$db,
 				'database.php',
 				'UPDATE ' . $this->db->quoteName('#__utf8_conversion')
-				. ' SET ' . $this->db->quoteName('converted') . ' = 0;'
+				. ' SET ' . $this->db->quoteName('converted') . ' = ' . $this->db->quoteName('converted') . ';'
 			);
 
 			// Set to not skipped
@@ -113,12 +113,12 @@ class ChangeSet
 			// Set the check query
 			if ($this->db instanceof UTF8MB4SupportInterface && $this->db->hasUTF8mb4Support())
 			{
-				$converted = 2;
+				$converted = 4;
 				$tmpSchemaChangeItem->queryType = 'UTF8_CONVERSION_UTF8MB4';
 			}
 			else
 			{
-				$converted = 1;
+				$converted = 3;
 				$tmpSchemaChangeItem->queryType = 'UTF8_CONVERSION_UTF8';
 			}
 

libraries/src/Table/Usergroup.php

@@ -243,6 +243,9 @@ class Usergroup extends Table
 		$db->setQuery($query);
 		$db->execute();
 
+		// Rebuild the nested set tree.
+		$this->rebuild();
+
 		// Delete the usergroup in view levels
 		$replace = array();