joomla
/
patchtester
mirror of https://github.com/joomla-extensions/patchtester.git
31
0
Fork 0
Code Issues Releases Activity

Broken Fetch Data down into chunks (Ref #27)

This commit is contained in:
Michael Babker 2015-02-22 20:49:59 -05:00
parent cbd5fcc1dc
commit d131a13f32
10 changed files with 296 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
administrator/components/com_patchtester/PatchTester/Controller/DisplayController.php
View File

@ -95,7 +95,15 @@ class DisplayController extends \JControllerBase
// Sanity check - Ensure our classes exist
if (!class_exists($viewClass))
{
throw new \RuntimeException(sprintf('The view class for the %1%s view in the %2%s was not found.', $view, $format), 500);
// Try to use a default view
$viewClass = '\\PatchTester\\View\\Default' . ucfirst($format) . 'View';
if (!class_exists($viewClass))
{
throw new \RuntimeException(
sprintf('The view class for the %1$s view in the %2$s was not found.', $view, $format), 500
);
}
}
if (!class_exists($modelClass))

40
administrator/components/com_patchtester/PatchTester/Controller/FetchController.php
View File

@ -26,26 +26,50 @@ class FetchController extends DisplayController
*/
public function execute()
{
// We don't want this request to be cached.
header('Pragma: no-cache');
header('Cache-Control: no-cache');
header('Expires: -1');
try
{
// Fetch our page from the session
$page = \JFactory::getSession()->get('com_patchtester_fetcher_page', 1);
// TODO - Decouple the model and context?
$model = new PullsModel('com_patchtester.fetch', null, \JFactory::getDbo());
// Initialize the state for the model
$model->setState($this->initializeState($model));
$model->requestFromGithub();
$msg = \JText::_('COM_PATCHTESTER_FETCH_SUCCESSFUL');
$type = 'message';
$status = $model->requestFromGithub($page);
}
catch (\Exception $e)
{
$msg = $e->getMessage();
$type = 'error';
$response = new \JResponseJson($e);
echo json_encode($response);
$this->getApplication()->close(1);
}
$this->getApplication()->enqueueMessage($msg, $type);
$this->getApplication()->redirect(\JRoute::_('index.php?option=com_patchtester', false));
// Update the UI and session now
if (isset($status['page']))
{
\JFactory::getSession()->set('com_patchtester_fetcher_page', $status['page']);
$message = \JText::sprintf('COM_PATCHTESTER_FETCH_PAGE_NUMBER', $status['page']);
unset($status['page']);
}
else
{
$status['header'] = \JText::_('COM_PATCHTESTER_FETCH_SUCCESSFUL', true);
$message = \JText::_('COM_PATCHTESTER_FETCH_COMPLETE_CLOSE_WINDOW', true);
}
$response = new \JResponseJson($status, $message, false, true);
echo json_encode($response);
$this->getApplication()->close();
}
}

105
administrator/components/com_patchtester/PatchTester/Controller/StartfetchController.php Normal file
View File

@ -0,0 +1,105 @@
<?php
/**
* Patch testing component for the Joomla! CMS
*
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
* @license GNU General Public License version 2 or later
*/
namespace PatchTester\Controller;
use PatchTester\Helper;
use PatchTester\Model\PullsModel;
/**
* Controller class to start fetching remote data
*
* @since 2.0
*/
class StartfetchController extends DisplayController
{
/**
* Execute the controller.
*
* @return void Redirects the application
*
* @since 2.0
*/
public function execute()
{
// We don't want this request to be cached.
header('Pragma: no-cache');
header('Cache-Control: no-cache');
header('Expires: -1');
// Check for a valid token. If invalid, send a 403 with the error message.
if (!\JSession::checkToken('request'))
{
$response = new \JResponseJson(new \Exception(\JText::_('JINVALID_TOKEN'), 403));
echo json_encode($response);
$this->getApplication()->close(1);
}
// Make sure we can fetch the data from GitHub - throw an error on < 10 available requests
$github = Helper::initializeGithub();
$rate = $github->authorization->getRateLimit();
// If over the API limit, we can't build this list
if ($rate->resources->core->remaining < 10)
{
$response = new \JResponseJson(
new \Exception(
\JText::sprintf('COM_PATCHTESTER_API_LIMIT_LIST', \JFactory::getDate($rate->resources->core->reset)),
429
)
);
echo json_encode($response);
$this->getApplication()->close(1);
}
// TODO - Decouple the model and context?
$model = new PullsModel('com_patchtester.fetch', null, \JFactory::getDbo());
// Initialize the state for the model
$model->setState($this->initializeState($model));
try
{
// Sanity check, ensure there aren't any applied patches
if (count($model->getAppliedPatches()) >= 1)
{
$response = new \JResponseJson(new \Exception(\JText::_('COM_PATCHTESTER_ERROR_APPLIED_PATCHES'), 500));
echo json_encode($response);
$this->getApplication()->close(1);
}
}
catch (\Exception $e)
{
$response = new \JResponseJson($e);
echo json_encode($response);
$this->getApplication()->close(1);
}
// We're able to successfully pull data, prepare our environment
\JFactory::getSession()->set('com_patchtester_fetcher_page', 1);
$response = new \JResponseJson(
array('complete' => false, 'header' => \JText::_('COM_PATCHTESTER_FETCH_PROCESSING', true)),
\JText::sprintf('COM_PATCHTESTER_FETCH_PAGE_NUMBER', 1),
false,
true
);
echo json_encode($response);
$this->getApplication()->close();
}
}

18
administrator/components/com_patchtester/PatchTester/Model/FetchModel.php Normal file
View File

@ -0,0 +1,18 @@
<?php
/**
* Patch testing component for the Joomla! CMS
*
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
* @license GNU General Public License version 2 or later
*/
namespace PatchTester\Model;
/**
* Model class for the fetch modal view
*
* @since 2.0
*/
class FetchModel extends PullsModel
{
}

64
administrator/components/com_patchtester/PatchTester/Model/PullsModel.php
View File

@ -299,63 +299,42 @@ class PullsModel extends \JModelDatabase
/**
* Method to request new data from GitHub
*
* @return void
* @param integer $page The page of the request
*
* @return array
*
* @since 2.0
* @throws \RuntimeException
*/
public function requestFromGithub()
public function requestFromGithub($page)
{
// Get the Github object
$github = Helper::initializeGithub();
$rate = $github->authorization->getRateLimit();
// If over the API limit, we can't build this list
if ($rate->resources->core->remaining == 0)
// If on page 1, dump the old data
if ($page === 1)
{
throw new \RuntimeException(
\JText::sprintf('COM_PATCHTESTER_API_LIMIT_LIST', \JFactory::getDate($rate->resources->core->reset))
$this->getDb()->truncateTable('#__patchtester_pulls');
}
try
{
// TODO - Option to configure the batch size
$pulls = $github->pulls->getList(
$this->getState()->get('github_user'), $this->getState()->get('github_repo'), 'open', $page, 100
);
}
// Sanity check, ensure there aren't any applied patches
if (count($this->getAppliedPatches()) >= 1)
catch (\DomainException $e)
{
throw new \RuntimeException(\JText::_('COM_PATCHTESTER_ERROR_APPLIED_PATCHES'));
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_GITHUB_FETCH', $e->getMessage()));
}
$pulls = array();
$page = 0;
$count = is_array($pulls) ? count($pulls) : 0;
do
// If there are no pulls to insert then bail, assume we're finished
if ($count === 0 || empty($pulls))
{
$page++;
try
{
$items = $github->pulls->getList($this->getState()->get('github_user'), $this->getState()->get('github_repo'), 'open', $page, 100);
}
catch (\DomainException $e)
{
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_GITHUB_FETCH', $e->getMessage()));
}
$count = is_array($items) ? count($items) : 0;
if ($count)
{
$pulls = array_merge($pulls, $items);
}
}
while ($count);
// Dump the old data now
$this->getDb()->truncateTable('#__patchtester_pulls');
// If there are no pulls to insert then bail
if (empty($pulls))
{
return;
return array('complete' => true);
}
$data = array();
@ -382,6 +361,9 @@ class PullsModel extends \JModelDatabase
{
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_INSERT_DATABASE', $e->getMessage()));
}
// Need to make another request
return array('complete' => false, 'page' => ($page + 1));
}
/**

20
administrator/components/com_patchtester/PatchTester/View/Fetch/tmpl/default.php Normal file
View File

@ -0,0 +1,20 @@
<?php
/**
* Patch testing component for the Joomla! CMS
*
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
* @license GNU General Public License version 2 or later
*/
/** @type \PatchTester\View\DefaultHtmlView $this */
JHtml::_('jquery.framework');
JHtml::_('script', 'com_patchtester/fetcher.js', false, true);
?>
<div id="patchtester-container">
<h1 id="patchtester-progress-header"><?php echo JText::_('COM_PATCHTESTER_FETCH_INITIALIZING'); ?></h1>
<p id="patchtester-progress-message"><?php echo JText::_('COM_PATCHTESTER_FETCH_INITIALIZING_DESCRIPTION'); ?></p>
<input id="patchtester-token" type="hidden" name="<?php echo JFactory::getSession()->getFormToken(); ?>" value="1" />
</div>

14
administrator/components/com_patchtester/PatchTester/View/Pulls/PullsHtmlView.php
View File

@ -102,7 +102,19 @@ class PullsHtmlView extends DefaultHtmlView
if (!count($this->envErrors))
{
\JToolbarHelper::custom('fetch', 'refresh.png', 'refresh_f2.png', 'COM_PATCHTESTER_TOOLBAR_FETCH_DATA', false);
$toolbar = \JToolbar::getInstance('toolbar');
$toolbar->appendButton(
'Popup',
'refresh',
'COM_PATCHTESTER_TOOLBAR_FETCH_DATA',
'index.php?option=com_patchtester&view=fetch&tmpl=component',
500,
210,
0,
0,
'window.parent.location.reload()',
'COM_PATCHTESTER_HEADING_FETCH_DATA'
);
}
\JToolBarHelper::preferences('com_patchtester');

9
administrator/components/com_patchtester/language/en-GB/en-GB.com_patchtester.ini
View File

@ -16,7 +16,13 @@ COM_PATCHTESTER_CONFLICT_S="The patch could not be applied because it conflicts
COM_PATCHTESTER_ERROR_APPLIED_PATCHES="Cannot fetch data from GitHub while there are applied patches. Please revert those patches before continuing."
COM_PATCHTESTER_ERROR_GITHUB_FETCH="Error retrieving pull requests from GitHub: %s"
COM_PATCHTESTER_ERROR_INSERT_DATABASE="Error inserting pull request data into the database: %s"
COM_PATCHTESTER_FETCH_SUCCESSFUL="Successfully retrieved pull requests."
COM_PATCHTESTER_FETCH_AN_ERROR_HAS_OCCURRED="An error has occurred while fetching the data from GitHub."
COM_PATCHTESTER_FETCH_COMPLETE_CLOSE_WINDOW="All data has been retrieved. Please close this modal window to refresh the page."
COM_PATCHTESTER_FETCH_INITIALIZING="Preparing to fetch GitHub data"
COM_PATCHTESTER_FETCH_INITIALIZING_DESCRIPTION="Making sure all is well to fetch data. Sit tight."
COM_PATCHTESTER_FETCH_PAGE_NUMBER="Processing page %s of GitHub data"
COM_PATCHTESTER_FETCH_PROCESSING="Processing data from GitHub"
COM_PATCHTESTER_FETCH_SUCCESSFUL="Successfully retrieved pull requests"
COM_PATCHTESTER_FIELD_GH_PASSWORD_LABEL="GitHub Account Password"
COM_PATCHTESTER_FIELD_GH_PASSWORD_DESC="Password for the account entered in the "_QQ_"GitHub Account"_QQ_" field. Note that accounts using Two Factor Authentication will not work with this component."
COM_PATCHTESTER_FIELD_GH_TOKEN_DESC="Use this field to input a GitHub API Token in place of your username and password. Note that this is required if your account has Two Factor Authentication enabled."
@ -31,6 +37,7 @@ COM_PATCHTESTER_FILE_DELETED_DOES_NOT_EXIST_S="The file marked for deletion does
COM_PATCHTESTER_FILE_MODIFIED_DOES_NOT_EXIST_S="The file marked for modification does not exist: %s"
COM_PATCHTESTER_FILTER_APPLIED_PATCHES="Filter Applied Patches"
COM_PATCHTESTER_FILTER_SEARCH_DESCRIPTION="Filter the list by title or ID."
COM_PATCHTESTER_HEADING_FETCH_DATA="Fetching GitHub Data"
COM_PATCHTESTER_PULL_ID="Pull ID"
COM_PATCHTESTER_NO_CREDENTIALS="No user credentials are saved, this will allow only 60 requests to the GitHub API per hour. Saving user credentials will allow 5,000 requests per hour."
COM_PATCHTESTER_NO_ITEMS="No data has been retrieved from GitHub, please click the 'Fetch Data' button in the toolbar to retrieve the open pull requests."

1
administrator/components/com_patchtester/patchtester.xml
View File

@ -35,6 +35,7 @@
<folder>css</folder>
<folder>fonts</folder>
<folder>images</folder>
<folder>js</folder>
</media>
<administration>
<menu img="../media/com_patchtester/images/icon-16-patchtester.png">com_patchtester</menu>

67
media/com_patchtester/js/fetcher.js Normal file
View File

@ -0,0 +1,67 @@
jQuery(function () {
var path = 'index.php?option=com_patchtester&tmpl=component&format=json';
function initialize() {
offset = 0;
progress = 0;
path = path + '&' + jQuery('#patchtester-token').attr('name') + '=1';
getRequest('startfetch');
};
function getRequest(task) {
jQuery.ajax({
type: "GET",
url: path,
data: 'task=' + task,
dataType: 'json',
success: handleResponse,
error: handleFailure
});
};
function handleResponse(json, resp) {
try {
if (json === null) {
throw resp;
}
if (json.error) {
throw json;
}
jQuery('#patchtester-progress-message').html(json.message);
if (json.data.header) {
jQuery('#patchtester-progress-header').html(json.data.header);
}
if (json.data.complete) {
// Nothing to do
} else {
// Send another request
getRequest('fetch');
}
} catch (error) {
try {
if (json.error) {
jQuery('#patchtester-progress-header').text(Joomla.JText._('COM_PATCHTESTER_FETCH_AN_ERROR_HAS_OCCURRED'));
jQuery('#patchtester-progress-message').html(json.message);
}
} catch (ignore) {
if (error === '') {
error = Joomla.JText._('COM_PATCHTESTER_NO_ERROR_RETURNED');
}
jQuery('#patchtester-progress-header').text(Joomla.JText._('COM_PATCHTESTER_FETCH_AN_ERROR_HAS_OCCURRED'));
jQuery('#patchtester-progress-message').html(error);
}
}
return true;
};
function handleFailure(xhr) {
json = (typeof xhr == 'object' && xhr.responseText) ? xhr.responseText : null;
jQuery('#patchtester-progress-header').text(Joomla.JText._('COM_PATCHTESTER_FETCH_AN_ERROR_HAS_OCCURRED'));
jQuery('#patchtester-progress-message').html(json);
};
initialize();
});