Add --keep-files-open-threshold (fixes #288)

This commit is contained in:
Jay Berkenbilt 2019-04-20 18:46:33 -04:00
parent 011695dfdf
commit e50d5201df
6 changed files with 57 additions and 327 deletions

View File

@ -1,5 +1,9 @@
2019-04-20 Jay Berkenbilt <ejb@ql.org>
* Add parameter --keep-files-open-threshold to override the
maximum number of files that qpdf will allow to be kept open at
once. Fixes #288.
* Handle Unicode characters in filenames. The changes to support
Unicode on the CLI in Windows broke Unicode filenames. Fixes #298.

View File

@ -652,6 +652,19 @@ make
more than the OS limit's number of files from a script and are
not worried about a few seconds additional processing time,
you may want to specify <option>--keep-files-open=n</option>.
The threshold for switching may be changed from the default
200 with the <option>--keep-files-open-threshold</option>
option.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--keep-files-open-threshold=<replaceable>count</replaceable></option></term>
<listitem>
<para>
If specified, overrides the default value of 200 used as the
threshold for qpdf deciding whether or not to keep files open.
See <option>--keep-files-open</option> for details.
</para>
</listitem>
</varlistentry>

View File

@ -134,6 +134,7 @@ struct Options
preserve_unreferenced_page_resources(false),
keep_files_open(true),
keep_files_open_set(false),
keep_files_open_threshold(200), // default known in help and docs
newline_before_endstream(false),
coalesce_contents(false),
flatten_annotations(false),
@ -225,6 +226,7 @@ struct Options
bool preserve_unreferenced_page_resources;
bool keep_files_open;
bool keep_files_open_set;
size_t keep_files_open_threshold;
bool newline_before_endstream;
std::string linearize_pass1;
bool coalesce_contents;
@ -634,6 +636,7 @@ class ArgParser
void argPreserveUnreferenced();
void argPreserveUnreferencedResources();
void argKeepFilesOpen(char* parameter);
void argKeepFilesOpenThreshold(char* parameter);
void argNewlineBeforeEndstream();
void argLinearizePass1(char* parameter);
void argCoalesceContents();
@ -857,6 +860,8 @@ ArgParser::initOptionTable()
&ArgParser::argPreserveUnreferencedResources);
(*t)["keep-files-open"] = oe_requiredChoices(
&ArgParser::argKeepFilesOpen, yn);
(*t)["keep-files-open-threshold"] = oe_requiredParameter(
&ArgParser::argKeepFilesOpenThreshold, "count");
(*t)["newline-before-endstream"] = oe_bare(
&ArgParser::argNewlineBeforeEndstream);
(*t)["linearize-pass1"] = oe_requiredParameter(
@ -1209,6 +1214,7 @@ ArgParser::argHelp()
<< "starting point, but its pages are replaced with pages as specified.\n"
<< "\n"
<< "--keep-files-open=[yn]\n"
<< "--keep-files-threshold=count\n"
<< "--pages file [ --password=password ] [ page-range ] ... --\n"
<< "\n"
<< "For each file that pages should be taken from, specify the file, a\n"
@ -1227,7 +1233,8 @@ ArgParser::argHelp()
<< "files will be kept open at the same time. This behavior can be overridden\n"
<< "by specifying --keep-files-open=[yn]. Closing and opening files can have\n"
<< "very high overhead on certain file systems, especially networked file\n"
<< "systems.\n"
<< "systems. The threshold of 200 can be modified with\n"
<< "--keep-files-open-threshold\n"
<< "\n"
<< "The page range is a set of numbers separated by commas, ranges of\n"
<< "numbers separated dashes, or combinations of those. The character\n"
@ -1781,6 +1788,13 @@ ArgParser::argKeepFilesOpen(char* parameter)
o.keep_files_open = (strcmp(parameter, "y") == 0);
}
void
ArgParser::argKeepFilesOpenThreshold(char* parameter)
{
o.keep_files_open_threshold =
static_cast<size_t>(QUtil::string_to_int(parameter));
}
void
ArgParser::argNewlineBeforeEndstream()
{
@ -4352,9 +4366,7 @@ static void handle_page_specs(QPDF& pdf, Options& o)
PageSpec& page_spec = *iter;
filenames.insert(page_spec.filename);
}
// NOTE: The number 200 for this threshold is in the help
// message and manual and is baked into the test suite.
if (filenames.size() > 200)
if (filenames.size() > o.keep_files_open_threshold)
{
QTC::TC("qpdf", "qpdf disable keep files open");
if (o.verbose)

View File

@ -1685,7 +1685,7 @@ $n_tests += 4;
binmode F;
my $content = <F>;
close(F);
for (my $i = 1; $i <= 201; ++$i)
for (my $i = 1; $i <= 51; ++$i)
{
open(F, sprintf(">%03d-kfo.pdf", $i)) or die;
binmode F;
@ -1693,16 +1693,17 @@ $n_tests += 4;
close(F);
}
}
$td->runtest("disable keep files open",
$td->runtest("automatic disable keep files open",
{$td->COMMAND =>
"qpdf --verbose --static-id --empty" .
" --keep-files-open-threshold=50" .
" --pages *kfo.pdf -- a.pdf"},
{$td->FILE => "disable-kfo.out", $td->EXIT_STATUS => 0},
$td->NORMALIZE_NEWLINES);
$td->runtest("don't disable keep files open",
{$td->COMMAND =>
"qpdf --verbose --static-id --empty" .
" --pages 11*kfo.pdf -- a.pdf"},
" --pages 01*kfo.pdf -- a.pdf"},
{$td->FILE => "enable-kfo.out", $td->EXIT_STATUS => 0},
$td->NORMALIZE_NEWLINES);
$td->runtest("explict keep files open",

View File

@ -50,156 +50,6 @@ qpdf: processing 048-kfo.pdf
qpdf: processing 049-kfo.pdf
qpdf: processing 050-kfo.pdf
qpdf: processing 051-kfo.pdf
qpdf: processing 052-kfo.pdf
qpdf: processing 053-kfo.pdf
qpdf: processing 054-kfo.pdf
qpdf: processing 055-kfo.pdf
qpdf: processing 056-kfo.pdf
qpdf: processing 057-kfo.pdf
qpdf: processing 058-kfo.pdf
qpdf: processing 059-kfo.pdf
qpdf: processing 060-kfo.pdf
qpdf: processing 061-kfo.pdf
qpdf: processing 062-kfo.pdf
qpdf: processing 063-kfo.pdf
qpdf: processing 064-kfo.pdf
qpdf: processing 065-kfo.pdf
qpdf: processing 066-kfo.pdf
qpdf: processing 067-kfo.pdf
qpdf: processing 068-kfo.pdf
qpdf: processing 069-kfo.pdf
qpdf: processing 070-kfo.pdf
qpdf: processing 071-kfo.pdf
qpdf: processing 072-kfo.pdf
qpdf: processing 073-kfo.pdf
qpdf: processing 074-kfo.pdf
qpdf: processing 075-kfo.pdf
qpdf: processing 076-kfo.pdf
qpdf: processing 077-kfo.pdf
qpdf: processing 078-kfo.pdf
qpdf: processing 079-kfo.pdf
qpdf: processing 080-kfo.pdf
qpdf: processing 081-kfo.pdf
qpdf: processing 082-kfo.pdf
qpdf: processing 083-kfo.pdf
qpdf: processing 084-kfo.pdf
qpdf: processing 085-kfo.pdf
qpdf: processing 086-kfo.pdf
qpdf: processing 087-kfo.pdf
qpdf: processing 088-kfo.pdf
qpdf: processing 089-kfo.pdf
qpdf: processing 090-kfo.pdf
qpdf: processing 091-kfo.pdf
qpdf: processing 092-kfo.pdf
qpdf: processing 093-kfo.pdf
qpdf: processing 094-kfo.pdf
qpdf: processing 095-kfo.pdf
qpdf: processing 096-kfo.pdf
qpdf: processing 097-kfo.pdf
qpdf: processing 098-kfo.pdf
qpdf: processing 099-kfo.pdf
qpdf: processing 100-kfo.pdf
qpdf: processing 101-kfo.pdf
qpdf: processing 102-kfo.pdf
qpdf: processing 103-kfo.pdf
qpdf: processing 104-kfo.pdf
qpdf: processing 105-kfo.pdf
qpdf: processing 106-kfo.pdf
qpdf: processing 107-kfo.pdf
qpdf: processing 108-kfo.pdf
qpdf: processing 109-kfo.pdf
qpdf: processing 110-kfo.pdf
qpdf: processing 111-kfo.pdf
qpdf: processing 112-kfo.pdf
qpdf: processing 113-kfo.pdf
qpdf: processing 114-kfo.pdf
qpdf: processing 115-kfo.pdf
qpdf: processing 116-kfo.pdf
qpdf: processing 117-kfo.pdf
qpdf: processing 118-kfo.pdf
qpdf: processing 119-kfo.pdf
qpdf: processing 120-kfo.pdf
qpdf: processing 121-kfo.pdf
qpdf: processing 122-kfo.pdf
qpdf: processing 123-kfo.pdf
qpdf: processing 124-kfo.pdf
qpdf: processing 125-kfo.pdf
qpdf: processing 126-kfo.pdf
qpdf: processing 127-kfo.pdf
qpdf: processing 128-kfo.pdf
qpdf: processing 129-kfo.pdf
qpdf: processing 130-kfo.pdf
qpdf: processing 131-kfo.pdf
qpdf: processing 132-kfo.pdf
qpdf: processing 133-kfo.pdf
qpdf: processing 134-kfo.pdf
qpdf: processing 135-kfo.pdf
qpdf: processing 136-kfo.pdf
qpdf: processing 137-kfo.pdf
qpdf: processing 138-kfo.pdf
qpdf: processing 139-kfo.pdf
qpdf: processing 140-kfo.pdf
qpdf: processing 141-kfo.pdf
qpdf: processing 142-kfo.pdf
qpdf: processing 143-kfo.pdf
qpdf: processing 144-kfo.pdf
qpdf: processing 145-kfo.pdf
qpdf: processing 146-kfo.pdf
qpdf: processing 147-kfo.pdf
qpdf: processing 148-kfo.pdf
qpdf: processing 149-kfo.pdf
qpdf: processing 150-kfo.pdf
qpdf: processing 151-kfo.pdf
qpdf: processing 152-kfo.pdf
qpdf: processing 153-kfo.pdf
qpdf: processing 154-kfo.pdf
qpdf: processing 155-kfo.pdf
qpdf: processing 156-kfo.pdf
qpdf: processing 157-kfo.pdf
qpdf: processing 158-kfo.pdf
qpdf: processing 159-kfo.pdf
qpdf: processing 160-kfo.pdf
qpdf: processing 161-kfo.pdf
qpdf: processing 162-kfo.pdf
qpdf: processing 163-kfo.pdf
qpdf: processing 164-kfo.pdf
qpdf: processing 165-kfo.pdf
qpdf: processing 166-kfo.pdf
qpdf: processing 167-kfo.pdf
qpdf: processing 168-kfo.pdf
qpdf: processing 169-kfo.pdf
qpdf: processing 170-kfo.pdf
qpdf: processing 171-kfo.pdf
qpdf: processing 172-kfo.pdf
qpdf: processing 173-kfo.pdf
qpdf: processing 174-kfo.pdf
qpdf: processing 175-kfo.pdf
qpdf: processing 176-kfo.pdf
qpdf: processing 177-kfo.pdf
qpdf: processing 178-kfo.pdf
qpdf: processing 179-kfo.pdf
qpdf: processing 180-kfo.pdf
qpdf: processing 181-kfo.pdf
qpdf: processing 182-kfo.pdf
qpdf: processing 183-kfo.pdf
qpdf: processing 184-kfo.pdf
qpdf: processing 185-kfo.pdf
qpdf: processing 186-kfo.pdf
qpdf: processing 187-kfo.pdf
qpdf: processing 188-kfo.pdf
qpdf: processing 189-kfo.pdf
qpdf: processing 190-kfo.pdf
qpdf: processing 191-kfo.pdf
qpdf: processing 192-kfo.pdf
qpdf: processing 193-kfo.pdf
qpdf: processing 194-kfo.pdf
qpdf: processing 195-kfo.pdf
qpdf: processing 196-kfo.pdf
qpdf: processing 197-kfo.pdf
qpdf: processing 198-kfo.pdf
qpdf: processing 199-kfo.pdf
qpdf: processing 200-kfo.pdf
qpdf: processing 201-kfo.pdf
qpdf: removing unreferenced pages from primary input
qpdf: adding pages from 001-kfo.pdf
qpdf: adding pages from 002-kfo.pdf
@ -252,154 +102,4 @@ qpdf: adding pages from 048-kfo.pdf
qpdf: adding pages from 049-kfo.pdf
qpdf: adding pages from 050-kfo.pdf
qpdf: adding pages from 051-kfo.pdf
qpdf: adding pages from 052-kfo.pdf
qpdf: adding pages from 053-kfo.pdf
qpdf: adding pages from 054-kfo.pdf
qpdf: adding pages from 055-kfo.pdf
qpdf: adding pages from 056-kfo.pdf
qpdf: adding pages from 057-kfo.pdf
qpdf: adding pages from 058-kfo.pdf
qpdf: adding pages from 059-kfo.pdf
qpdf: adding pages from 060-kfo.pdf
qpdf: adding pages from 061-kfo.pdf
qpdf: adding pages from 062-kfo.pdf
qpdf: adding pages from 063-kfo.pdf
qpdf: adding pages from 064-kfo.pdf
qpdf: adding pages from 065-kfo.pdf
qpdf: adding pages from 066-kfo.pdf
qpdf: adding pages from 067-kfo.pdf
qpdf: adding pages from 068-kfo.pdf
qpdf: adding pages from 069-kfo.pdf
qpdf: adding pages from 070-kfo.pdf
qpdf: adding pages from 071-kfo.pdf
qpdf: adding pages from 072-kfo.pdf
qpdf: adding pages from 073-kfo.pdf
qpdf: adding pages from 074-kfo.pdf
qpdf: adding pages from 075-kfo.pdf
qpdf: adding pages from 076-kfo.pdf
qpdf: adding pages from 077-kfo.pdf
qpdf: adding pages from 078-kfo.pdf
qpdf: adding pages from 079-kfo.pdf
qpdf: adding pages from 080-kfo.pdf
qpdf: adding pages from 081-kfo.pdf
qpdf: adding pages from 082-kfo.pdf
qpdf: adding pages from 083-kfo.pdf
qpdf: adding pages from 084-kfo.pdf
qpdf: adding pages from 085-kfo.pdf
qpdf: adding pages from 086-kfo.pdf
qpdf: adding pages from 087-kfo.pdf
qpdf: adding pages from 088-kfo.pdf
qpdf: adding pages from 089-kfo.pdf
qpdf: adding pages from 090-kfo.pdf
qpdf: adding pages from 091-kfo.pdf
qpdf: adding pages from 092-kfo.pdf
qpdf: adding pages from 093-kfo.pdf
qpdf: adding pages from 094-kfo.pdf
qpdf: adding pages from 095-kfo.pdf
qpdf: adding pages from 096-kfo.pdf
qpdf: adding pages from 097-kfo.pdf
qpdf: adding pages from 098-kfo.pdf
qpdf: adding pages from 099-kfo.pdf
qpdf: adding pages from 100-kfo.pdf
qpdf: adding pages from 101-kfo.pdf
qpdf: adding pages from 102-kfo.pdf
qpdf: adding pages from 103-kfo.pdf
qpdf: adding pages from 104-kfo.pdf
qpdf: adding pages from 105-kfo.pdf
qpdf: adding pages from 106-kfo.pdf
qpdf: adding pages from 107-kfo.pdf
qpdf: adding pages from 108-kfo.pdf
qpdf: adding pages from 109-kfo.pdf
qpdf: adding pages from 110-kfo.pdf
qpdf: adding pages from 111-kfo.pdf
qpdf: adding pages from 112-kfo.pdf
qpdf: adding pages from 113-kfo.pdf
qpdf: adding pages from 114-kfo.pdf
qpdf: adding pages from 115-kfo.pdf
qpdf: adding pages from 116-kfo.pdf
qpdf: adding pages from 117-kfo.pdf
qpdf: adding pages from 118-kfo.pdf
qpdf: adding pages from 119-kfo.pdf
qpdf: adding pages from 120-kfo.pdf
qpdf: adding pages from 121-kfo.pdf
qpdf: adding pages from 122-kfo.pdf
qpdf: adding pages from 123-kfo.pdf
qpdf: adding pages from 124-kfo.pdf
qpdf: adding pages from 125-kfo.pdf
qpdf: adding pages from 126-kfo.pdf
qpdf: adding pages from 127-kfo.pdf
qpdf: adding pages from 128-kfo.pdf
qpdf: adding pages from 129-kfo.pdf
qpdf: adding pages from 130-kfo.pdf
qpdf: adding pages from 131-kfo.pdf
qpdf: adding pages from 132-kfo.pdf
qpdf: adding pages from 133-kfo.pdf
qpdf: adding pages from 134-kfo.pdf
qpdf: adding pages from 135-kfo.pdf
qpdf: adding pages from 136-kfo.pdf
qpdf: adding pages from 137-kfo.pdf
qpdf: adding pages from 138-kfo.pdf
qpdf: adding pages from 139-kfo.pdf
qpdf: adding pages from 140-kfo.pdf
qpdf: adding pages from 141-kfo.pdf
qpdf: adding pages from 142-kfo.pdf
qpdf: adding pages from 143-kfo.pdf
qpdf: adding pages from 144-kfo.pdf
qpdf: adding pages from 145-kfo.pdf
qpdf: adding pages from 146-kfo.pdf
qpdf: adding pages from 147-kfo.pdf
qpdf: adding pages from 148-kfo.pdf
qpdf: adding pages from 149-kfo.pdf
qpdf: adding pages from 150-kfo.pdf
qpdf: adding pages from 151-kfo.pdf
qpdf: adding pages from 152-kfo.pdf
qpdf: adding pages from 153-kfo.pdf
qpdf: adding pages from 154-kfo.pdf
qpdf: adding pages from 155-kfo.pdf
qpdf: adding pages from 156-kfo.pdf
qpdf: adding pages from 157-kfo.pdf
qpdf: adding pages from 158-kfo.pdf
qpdf: adding pages from 159-kfo.pdf
qpdf: adding pages from 160-kfo.pdf
qpdf: adding pages from 161-kfo.pdf
qpdf: adding pages from 162-kfo.pdf
qpdf: adding pages from 163-kfo.pdf
qpdf: adding pages from 164-kfo.pdf
qpdf: adding pages from 165-kfo.pdf
qpdf: adding pages from 166-kfo.pdf
qpdf: adding pages from 167-kfo.pdf
qpdf: adding pages from 168-kfo.pdf
qpdf: adding pages from 169-kfo.pdf
qpdf: adding pages from 170-kfo.pdf
qpdf: adding pages from 171-kfo.pdf
qpdf: adding pages from 172-kfo.pdf
qpdf: adding pages from 173-kfo.pdf
qpdf: adding pages from 174-kfo.pdf
qpdf: adding pages from 175-kfo.pdf
qpdf: adding pages from 176-kfo.pdf
qpdf: adding pages from 177-kfo.pdf
qpdf: adding pages from 178-kfo.pdf
qpdf: adding pages from 179-kfo.pdf
qpdf: adding pages from 180-kfo.pdf
qpdf: adding pages from 181-kfo.pdf
qpdf: adding pages from 182-kfo.pdf
qpdf: adding pages from 183-kfo.pdf
qpdf: adding pages from 184-kfo.pdf
qpdf: adding pages from 185-kfo.pdf
qpdf: adding pages from 186-kfo.pdf
qpdf: adding pages from 187-kfo.pdf
qpdf: adding pages from 188-kfo.pdf
qpdf: adding pages from 189-kfo.pdf
qpdf: adding pages from 190-kfo.pdf
qpdf: adding pages from 191-kfo.pdf
qpdf: adding pages from 192-kfo.pdf
qpdf: adding pages from 193-kfo.pdf
qpdf: adding pages from 194-kfo.pdf
qpdf: adding pages from 195-kfo.pdf
qpdf: adding pages from 196-kfo.pdf
qpdf: adding pages from 197-kfo.pdf
qpdf: adding pages from 198-kfo.pdf
qpdf: adding pages from 199-kfo.pdf
qpdf: adding pages from 200-kfo.pdf
qpdf: adding pages from 201-kfo.pdf
qpdf: wrote file a.pdf

View File

@ -1,23 +1,23 @@
qpdf: selecting --keep-open-files=y
qpdf: processing 110-kfo.pdf
qpdf: processing 111-kfo.pdf
qpdf: processing 112-kfo.pdf
qpdf: processing 113-kfo.pdf
qpdf: processing 114-kfo.pdf
qpdf: processing 115-kfo.pdf
qpdf: processing 116-kfo.pdf
qpdf: processing 117-kfo.pdf
qpdf: processing 118-kfo.pdf
qpdf: processing 119-kfo.pdf
qpdf: processing 010-kfo.pdf
qpdf: processing 011-kfo.pdf
qpdf: processing 012-kfo.pdf
qpdf: processing 013-kfo.pdf
qpdf: processing 014-kfo.pdf
qpdf: processing 015-kfo.pdf
qpdf: processing 016-kfo.pdf
qpdf: processing 017-kfo.pdf
qpdf: processing 018-kfo.pdf
qpdf: processing 019-kfo.pdf
qpdf: removing unreferenced pages from primary input
qpdf: adding pages from 110-kfo.pdf
qpdf: adding pages from 111-kfo.pdf
qpdf: adding pages from 112-kfo.pdf
qpdf: adding pages from 113-kfo.pdf
qpdf: adding pages from 114-kfo.pdf
qpdf: adding pages from 115-kfo.pdf
qpdf: adding pages from 116-kfo.pdf
qpdf: adding pages from 117-kfo.pdf
qpdf: adding pages from 118-kfo.pdf
qpdf: adding pages from 119-kfo.pdf
qpdf: adding pages from 010-kfo.pdf
qpdf: adding pages from 011-kfo.pdf
qpdf: adding pages from 012-kfo.pdf
qpdf: adding pages from 013-kfo.pdf
qpdf: adding pages from 014-kfo.pdf
qpdf: adding pages from 015-kfo.pdf
qpdf: adding pages from 016-kfo.pdf
qpdf: adding pages from 017-kfo.pdf
qpdf: adding pages from 018-kfo.pdf
qpdf: adding pages from 019-kfo.pdf
qpdf: wrote file a.pdf