From e50d5201dfce53158ba811865e99654054f0c872 Mon Sep 17 00:00:00 2001 From: Jay Berkenbilt Date: Sat, 20 Apr 2019 18:46:33 -0400 Subject: [PATCH] Add --keep-files-open-threshold (fixes #288) --- ChangeLog | 4 + manual/qpdf-manual.xml | 13 ++ qpdf/qpdf.cc | 20 ++- qpdf/qtest/qpdf.test | 7 +- qpdf/qtest/qpdf/disable-kfo.out | 300 -------------------------------- qpdf/qtest/qpdf/enable-kfo.out | 40 ++--- 6 files changed, 57 insertions(+), 327 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8514e379..944d80b4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ 2019-04-20 Jay Berkenbilt + * Add parameter --keep-files-open-threshold to override the + maximum number of files that qpdf will allow to be kept open at + once. Fixes #288. + * Handle Unicode characters in filenames. The changes to support Unicode on the CLI in Windows broke Unicode filenames. Fixes #298. diff --git a/manual/qpdf-manual.xml b/manual/qpdf-manual.xml index c2df34bf..669fd42e 100644 --- a/manual/qpdf-manual.xml +++ b/manual/qpdf-manual.xml @@ -652,6 +652,19 @@ make more than the OS limit's number of files from a script and are not worried about a few seconds additional processing time, you may want to specify . + The threshold for switching may be changed from the default + 200 with the + option. + + + + + + + + If specified, overrides the default value of 200 used as the + threshold for qpdf deciding whether or not to keep files open. + See for details. diff --git a/qpdf/qpdf.cc b/qpdf/qpdf.cc index 86c0b7a5..3d80abac 100644 --- a/qpdf/qpdf.cc +++ b/qpdf/qpdf.cc @@ -134,6 +134,7 @@ struct Options preserve_unreferenced_page_resources(false), keep_files_open(true), keep_files_open_set(false), + keep_files_open_threshold(200), // default known in help and docs newline_before_endstream(false), coalesce_contents(false), flatten_annotations(false), @@ -225,6 +226,7 @@ struct Options bool preserve_unreferenced_page_resources; bool keep_files_open; bool keep_files_open_set; + size_t keep_files_open_threshold; bool newline_before_endstream; std::string linearize_pass1; bool coalesce_contents; @@ -634,6 +636,7 @@ class ArgParser void argPreserveUnreferenced(); void argPreserveUnreferencedResources(); void argKeepFilesOpen(char* parameter); + void argKeepFilesOpenThreshold(char* parameter); void argNewlineBeforeEndstream(); void argLinearizePass1(char* parameter); void argCoalesceContents(); @@ -857,6 +860,8 @@ ArgParser::initOptionTable() &ArgParser::argPreserveUnreferencedResources); (*t)["keep-files-open"] = oe_requiredChoices( &ArgParser::argKeepFilesOpen, yn); + (*t)["keep-files-open-threshold"] = oe_requiredParameter( + &ArgParser::argKeepFilesOpenThreshold, "count"); (*t)["newline-before-endstream"] = oe_bare( &ArgParser::argNewlineBeforeEndstream); (*t)["linearize-pass1"] = oe_requiredParameter( @@ -1209,6 +1214,7 @@ ArgParser::argHelp() << "starting point, but its pages are replaced with pages as specified.\n" << "\n" << "--keep-files-open=[yn]\n" + << "--keep-files-threshold=count\n" << "--pages file [ --password=password ] [ page-range ] ... --\n" << "\n" << "For each file that pages should be taken from, specify the file, a\n" @@ -1227,7 +1233,8 @@ ArgParser::argHelp() << "files will be kept open at the same time. This behavior can be overridden\n" << "by specifying --keep-files-open=[yn]. Closing and opening files can have\n" << "very high overhead on certain file systems, especially networked file\n" - << "systems.\n" + << "systems. The threshold of 200 can be modified with\n" + << "--keep-files-open-threshold\n" << "\n" << "The page range is a set of numbers separated by commas, ranges of\n" << "numbers separated dashes, or combinations of those. The character\n" @@ -1781,6 +1788,13 @@ ArgParser::argKeepFilesOpen(char* parameter) o.keep_files_open = (strcmp(parameter, "y") == 0); } +void +ArgParser::argKeepFilesOpenThreshold(char* parameter) +{ + o.keep_files_open_threshold = + static_cast(QUtil::string_to_int(parameter)); +} + void ArgParser::argNewlineBeforeEndstream() { @@ -4352,9 +4366,7 @@ static void handle_page_specs(QPDF& pdf, Options& o) PageSpec& page_spec = *iter; filenames.insert(page_spec.filename); } - // NOTE: The number 200 for this threshold is in the help - // message and manual and is baked into the test suite. - if (filenames.size() > 200) + if (filenames.size() > o.keep_files_open_threshold) { QTC::TC("qpdf", "qpdf disable keep files open"); if (o.verbose) diff --git a/qpdf/qtest/qpdf.test b/qpdf/qtest/qpdf.test index 4dd0909f..d01ac3af 100644 --- a/qpdf/qtest/qpdf.test +++ b/qpdf/qtest/qpdf.test @@ -1685,7 +1685,7 @@ $n_tests += 4; binmode F; my $content = ; close(F); - for (my $i = 1; $i <= 201; ++$i) + for (my $i = 1; $i <= 51; ++$i) { open(F, sprintf(">%03d-kfo.pdf", $i)) or die; binmode F; @@ -1693,16 +1693,17 @@ $n_tests += 4; close(F); } } -$td->runtest("disable keep files open", +$td->runtest("automatic disable keep files open", {$td->COMMAND => "qpdf --verbose --static-id --empty" . + " --keep-files-open-threshold=50" . " --pages *kfo.pdf -- a.pdf"}, {$td->FILE => "disable-kfo.out", $td->EXIT_STATUS => 0}, $td->NORMALIZE_NEWLINES); $td->runtest("don't disable keep files open", {$td->COMMAND => "qpdf --verbose --static-id --empty" . - " --pages 11*kfo.pdf -- a.pdf"}, + " --pages 01*kfo.pdf -- a.pdf"}, {$td->FILE => "enable-kfo.out", $td->EXIT_STATUS => 0}, $td->NORMALIZE_NEWLINES); $td->runtest("explict keep files open", diff --git a/qpdf/qtest/qpdf/disable-kfo.out b/qpdf/qtest/qpdf/disable-kfo.out index 4aec24b6..a33044e0 100644 --- a/qpdf/qtest/qpdf/disable-kfo.out +++ b/qpdf/qtest/qpdf/disable-kfo.out @@ -50,156 +50,6 @@ qpdf: processing 048-kfo.pdf qpdf: processing 049-kfo.pdf qpdf: processing 050-kfo.pdf qpdf: processing 051-kfo.pdf -qpdf: processing 052-kfo.pdf -qpdf: processing 053-kfo.pdf -qpdf: processing 054-kfo.pdf -qpdf: processing 055-kfo.pdf -qpdf: processing 056-kfo.pdf -qpdf: processing 057-kfo.pdf -qpdf: processing 058-kfo.pdf -qpdf: processing 059-kfo.pdf -qpdf: processing 060-kfo.pdf -qpdf: processing 061-kfo.pdf -qpdf: processing 062-kfo.pdf -qpdf: processing 063-kfo.pdf -qpdf: processing 064-kfo.pdf -qpdf: processing 065-kfo.pdf -qpdf: processing 066-kfo.pdf -qpdf: processing 067-kfo.pdf -qpdf: processing 068-kfo.pdf -qpdf: processing 069-kfo.pdf -qpdf: processing 070-kfo.pdf -qpdf: processing 071-kfo.pdf -qpdf: processing 072-kfo.pdf -qpdf: processing 073-kfo.pdf -qpdf: processing 074-kfo.pdf -qpdf: processing 075-kfo.pdf -qpdf: processing 076-kfo.pdf -qpdf: processing 077-kfo.pdf -qpdf: processing 078-kfo.pdf -qpdf: processing 079-kfo.pdf -qpdf: processing 080-kfo.pdf -qpdf: processing 081-kfo.pdf -qpdf: processing 082-kfo.pdf -qpdf: processing 083-kfo.pdf -qpdf: processing 084-kfo.pdf -qpdf: processing 085-kfo.pdf -qpdf: processing 086-kfo.pdf -qpdf: processing 087-kfo.pdf -qpdf: processing 088-kfo.pdf -qpdf: processing 089-kfo.pdf -qpdf: processing 090-kfo.pdf -qpdf: processing 091-kfo.pdf -qpdf: processing 092-kfo.pdf -qpdf: processing 093-kfo.pdf -qpdf: processing 094-kfo.pdf -qpdf: processing 095-kfo.pdf -qpdf: processing 096-kfo.pdf -qpdf: processing 097-kfo.pdf -qpdf: processing 098-kfo.pdf -qpdf: processing 099-kfo.pdf -qpdf: processing 100-kfo.pdf -qpdf: processing 101-kfo.pdf -qpdf: processing 102-kfo.pdf -qpdf: processing 103-kfo.pdf -qpdf: processing 104-kfo.pdf -qpdf: processing 105-kfo.pdf -qpdf: processing 106-kfo.pdf -qpdf: processing 107-kfo.pdf -qpdf: processing 108-kfo.pdf -qpdf: processing 109-kfo.pdf -qpdf: processing 110-kfo.pdf -qpdf: processing 111-kfo.pdf -qpdf: processing 112-kfo.pdf -qpdf: processing 113-kfo.pdf -qpdf: processing 114-kfo.pdf -qpdf: processing 115-kfo.pdf -qpdf: processing 116-kfo.pdf -qpdf: processing 117-kfo.pdf -qpdf: processing 118-kfo.pdf -qpdf: processing 119-kfo.pdf -qpdf: processing 120-kfo.pdf -qpdf: processing 121-kfo.pdf -qpdf: processing 122-kfo.pdf -qpdf: processing 123-kfo.pdf -qpdf: processing 124-kfo.pdf -qpdf: processing 125-kfo.pdf -qpdf: processing 126-kfo.pdf -qpdf: processing 127-kfo.pdf -qpdf: processing 128-kfo.pdf -qpdf: processing 129-kfo.pdf -qpdf: processing 130-kfo.pdf -qpdf: processing 131-kfo.pdf -qpdf: processing 132-kfo.pdf -qpdf: processing 133-kfo.pdf -qpdf: processing 134-kfo.pdf -qpdf: processing 135-kfo.pdf -qpdf: processing 136-kfo.pdf -qpdf: processing 137-kfo.pdf -qpdf: processing 138-kfo.pdf -qpdf: processing 139-kfo.pdf -qpdf: processing 140-kfo.pdf -qpdf: processing 141-kfo.pdf -qpdf: processing 142-kfo.pdf -qpdf: processing 143-kfo.pdf -qpdf: processing 144-kfo.pdf -qpdf: processing 145-kfo.pdf -qpdf: processing 146-kfo.pdf -qpdf: processing 147-kfo.pdf -qpdf: processing 148-kfo.pdf -qpdf: processing 149-kfo.pdf -qpdf: processing 150-kfo.pdf -qpdf: processing 151-kfo.pdf -qpdf: processing 152-kfo.pdf -qpdf: processing 153-kfo.pdf -qpdf: processing 154-kfo.pdf -qpdf: processing 155-kfo.pdf -qpdf: processing 156-kfo.pdf -qpdf: processing 157-kfo.pdf -qpdf: processing 158-kfo.pdf -qpdf: processing 159-kfo.pdf -qpdf: processing 160-kfo.pdf -qpdf: processing 161-kfo.pdf -qpdf: processing 162-kfo.pdf -qpdf: processing 163-kfo.pdf -qpdf: processing 164-kfo.pdf -qpdf: processing 165-kfo.pdf -qpdf: processing 166-kfo.pdf -qpdf: processing 167-kfo.pdf -qpdf: processing 168-kfo.pdf -qpdf: processing 169-kfo.pdf -qpdf: processing 170-kfo.pdf -qpdf: processing 171-kfo.pdf -qpdf: processing 172-kfo.pdf -qpdf: processing 173-kfo.pdf -qpdf: processing 174-kfo.pdf -qpdf: processing 175-kfo.pdf -qpdf: processing 176-kfo.pdf -qpdf: processing 177-kfo.pdf -qpdf: processing 178-kfo.pdf -qpdf: processing 179-kfo.pdf -qpdf: processing 180-kfo.pdf -qpdf: processing 181-kfo.pdf -qpdf: processing 182-kfo.pdf -qpdf: processing 183-kfo.pdf -qpdf: processing 184-kfo.pdf -qpdf: processing 185-kfo.pdf -qpdf: processing 186-kfo.pdf -qpdf: processing 187-kfo.pdf -qpdf: processing 188-kfo.pdf -qpdf: processing 189-kfo.pdf -qpdf: processing 190-kfo.pdf -qpdf: processing 191-kfo.pdf -qpdf: processing 192-kfo.pdf -qpdf: processing 193-kfo.pdf -qpdf: processing 194-kfo.pdf -qpdf: processing 195-kfo.pdf -qpdf: processing 196-kfo.pdf -qpdf: processing 197-kfo.pdf -qpdf: processing 198-kfo.pdf -qpdf: processing 199-kfo.pdf -qpdf: processing 200-kfo.pdf -qpdf: processing 201-kfo.pdf qpdf: removing unreferenced pages from primary input qpdf: adding pages from 001-kfo.pdf qpdf: adding pages from 002-kfo.pdf @@ -252,154 +102,4 @@ qpdf: adding pages from 048-kfo.pdf qpdf: adding pages from 049-kfo.pdf qpdf: adding pages from 050-kfo.pdf qpdf: adding pages from 051-kfo.pdf -qpdf: adding pages from 052-kfo.pdf -qpdf: adding pages from 053-kfo.pdf -qpdf: adding pages from 054-kfo.pdf -qpdf: adding pages from 055-kfo.pdf -qpdf: adding pages from 056-kfo.pdf -qpdf: adding pages from 057-kfo.pdf -qpdf: adding pages from 058-kfo.pdf -qpdf: adding pages from 059-kfo.pdf -qpdf: adding pages from 060-kfo.pdf -qpdf: adding pages from 061-kfo.pdf -qpdf: adding pages from 062-kfo.pdf -qpdf: adding pages from 063-kfo.pdf -qpdf: adding pages from 064-kfo.pdf -qpdf: adding pages from 065-kfo.pdf -qpdf: adding pages from 066-kfo.pdf -qpdf: adding pages from 067-kfo.pdf -qpdf: adding pages from 068-kfo.pdf -qpdf: adding pages from 069-kfo.pdf -qpdf: adding pages from 070-kfo.pdf -qpdf: adding pages from 071-kfo.pdf -qpdf: adding pages from 072-kfo.pdf -qpdf: adding pages from 073-kfo.pdf -qpdf: adding pages from 074-kfo.pdf -qpdf: adding pages from 075-kfo.pdf -qpdf: adding pages from 076-kfo.pdf -qpdf: adding pages from 077-kfo.pdf -qpdf: adding pages from 078-kfo.pdf -qpdf: adding pages from 079-kfo.pdf -qpdf: adding pages from 080-kfo.pdf -qpdf: adding pages from 081-kfo.pdf -qpdf: adding pages from 082-kfo.pdf -qpdf: adding pages from 083-kfo.pdf -qpdf: adding pages from 084-kfo.pdf -qpdf: adding pages from 085-kfo.pdf -qpdf: adding pages from 086-kfo.pdf -qpdf: adding pages from 087-kfo.pdf -qpdf: adding pages from 088-kfo.pdf -qpdf: adding pages from 089-kfo.pdf -qpdf: adding pages from 090-kfo.pdf -qpdf: adding pages from 091-kfo.pdf -qpdf: adding pages from 092-kfo.pdf -qpdf: adding pages from 093-kfo.pdf -qpdf: adding pages from 094-kfo.pdf -qpdf: adding pages from 095-kfo.pdf -qpdf: adding pages from 096-kfo.pdf -qpdf: adding pages from 097-kfo.pdf -qpdf: adding pages from 098-kfo.pdf -qpdf: adding pages from 099-kfo.pdf -qpdf: adding pages from 100-kfo.pdf -qpdf: adding pages from 101-kfo.pdf -qpdf: adding pages from 102-kfo.pdf -qpdf: adding pages from 103-kfo.pdf -qpdf: adding pages from 104-kfo.pdf -qpdf: adding pages from 105-kfo.pdf -qpdf: adding pages from 106-kfo.pdf -qpdf: adding pages from 107-kfo.pdf -qpdf: adding pages from 108-kfo.pdf -qpdf: adding pages from 109-kfo.pdf -qpdf: adding pages from 110-kfo.pdf -qpdf: adding pages from 111-kfo.pdf -qpdf: adding pages from 112-kfo.pdf -qpdf: adding pages from 113-kfo.pdf -qpdf: adding pages from 114-kfo.pdf -qpdf: adding pages from 115-kfo.pdf -qpdf: adding pages from 116-kfo.pdf -qpdf: adding pages from 117-kfo.pdf -qpdf: adding pages from 118-kfo.pdf -qpdf: adding pages from 119-kfo.pdf -qpdf: adding pages from 120-kfo.pdf -qpdf: adding pages from 121-kfo.pdf -qpdf: adding pages from 122-kfo.pdf -qpdf: adding pages from 123-kfo.pdf -qpdf: adding pages from 124-kfo.pdf -qpdf: adding pages from 125-kfo.pdf -qpdf: adding pages from 126-kfo.pdf -qpdf: adding pages from 127-kfo.pdf -qpdf: adding pages from 128-kfo.pdf -qpdf: adding pages from 129-kfo.pdf -qpdf: adding pages from 130-kfo.pdf -qpdf: adding pages from 131-kfo.pdf -qpdf: adding pages from 132-kfo.pdf -qpdf: adding pages from 133-kfo.pdf -qpdf: adding pages from 134-kfo.pdf -qpdf: adding pages from 135-kfo.pdf -qpdf: adding pages from 136-kfo.pdf -qpdf: adding pages from 137-kfo.pdf -qpdf: adding pages from 138-kfo.pdf -qpdf: adding pages from 139-kfo.pdf -qpdf: adding pages from 140-kfo.pdf -qpdf: adding pages from 141-kfo.pdf -qpdf: adding pages from 142-kfo.pdf -qpdf: adding pages from 143-kfo.pdf -qpdf: adding pages from 144-kfo.pdf -qpdf: adding pages from 145-kfo.pdf -qpdf: adding pages from 146-kfo.pdf -qpdf: adding pages from 147-kfo.pdf -qpdf: adding pages from 148-kfo.pdf -qpdf: adding pages from 149-kfo.pdf -qpdf: adding pages from 150-kfo.pdf -qpdf: adding pages from 151-kfo.pdf -qpdf: adding pages from 152-kfo.pdf -qpdf: adding pages from 153-kfo.pdf -qpdf: adding pages from 154-kfo.pdf -qpdf: adding pages from 155-kfo.pdf -qpdf: adding pages from 156-kfo.pdf -qpdf: adding pages from 157-kfo.pdf -qpdf: adding pages from 158-kfo.pdf -qpdf: adding pages from 159-kfo.pdf -qpdf: adding pages from 160-kfo.pdf -qpdf: adding pages from 161-kfo.pdf -qpdf: adding pages from 162-kfo.pdf -qpdf: adding pages from 163-kfo.pdf -qpdf: adding pages from 164-kfo.pdf -qpdf: adding pages from 165-kfo.pdf -qpdf: adding pages from 166-kfo.pdf -qpdf: adding pages from 167-kfo.pdf -qpdf: adding pages from 168-kfo.pdf -qpdf: adding pages from 169-kfo.pdf -qpdf: adding pages from 170-kfo.pdf -qpdf: adding pages from 171-kfo.pdf -qpdf: adding pages from 172-kfo.pdf -qpdf: adding pages from 173-kfo.pdf -qpdf: adding pages from 174-kfo.pdf -qpdf: adding pages from 175-kfo.pdf -qpdf: adding pages from 176-kfo.pdf -qpdf: adding pages from 177-kfo.pdf -qpdf: adding pages from 178-kfo.pdf -qpdf: adding pages from 179-kfo.pdf -qpdf: adding pages from 180-kfo.pdf -qpdf: adding pages from 181-kfo.pdf -qpdf: adding pages from 182-kfo.pdf -qpdf: adding pages from 183-kfo.pdf -qpdf: adding pages from 184-kfo.pdf -qpdf: adding pages from 185-kfo.pdf -qpdf: adding pages from 186-kfo.pdf -qpdf: adding pages from 187-kfo.pdf -qpdf: adding pages from 188-kfo.pdf -qpdf: adding pages from 189-kfo.pdf -qpdf: adding pages from 190-kfo.pdf -qpdf: adding pages from 191-kfo.pdf -qpdf: adding pages from 192-kfo.pdf -qpdf: adding pages from 193-kfo.pdf -qpdf: adding pages from 194-kfo.pdf -qpdf: adding pages from 195-kfo.pdf -qpdf: adding pages from 196-kfo.pdf -qpdf: adding pages from 197-kfo.pdf -qpdf: adding pages from 198-kfo.pdf -qpdf: adding pages from 199-kfo.pdf -qpdf: adding pages from 200-kfo.pdf -qpdf: adding pages from 201-kfo.pdf qpdf: wrote file a.pdf diff --git a/qpdf/qtest/qpdf/enable-kfo.out b/qpdf/qtest/qpdf/enable-kfo.out index 82e1b7c4..0c011518 100644 --- a/qpdf/qtest/qpdf/enable-kfo.out +++ b/qpdf/qtest/qpdf/enable-kfo.out @@ -1,23 +1,23 @@ qpdf: selecting --keep-open-files=y -qpdf: processing 110-kfo.pdf -qpdf: processing 111-kfo.pdf -qpdf: processing 112-kfo.pdf -qpdf: processing 113-kfo.pdf -qpdf: processing 114-kfo.pdf -qpdf: processing 115-kfo.pdf -qpdf: processing 116-kfo.pdf -qpdf: processing 117-kfo.pdf -qpdf: processing 118-kfo.pdf -qpdf: processing 119-kfo.pdf +qpdf: processing 010-kfo.pdf +qpdf: processing 011-kfo.pdf +qpdf: processing 012-kfo.pdf +qpdf: processing 013-kfo.pdf +qpdf: processing 014-kfo.pdf +qpdf: processing 015-kfo.pdf +qpdf: processing 016-kfo.pdf +qpdf: processing 017-kfo.pdf +qpdf: processing 018-kfo.pdf +qpdf: processing 019-kfo.pdf qpdf: removing unreferenced pages from primary input -qpdf: adding pages from 110-kfo.pdf -qpdf: adding pages from 111-kfo.pdf -qpdf: adding pages from 112-kfo.pdf -qpdf: adding pages from 113-kfo.pdf -qpdf: adding pages from 114-kfo.pdf -qpdf: adding pages from 115-kfo.pdf -qpdf: adding pages from 116-kfo.pdf -qpdf: adding pages from 117-kfo.pdf -qpdf: adding pages from 118-kfo.pdf -qpdf: adding pages from 119-kfo.pdf +qpdf: adding pages from 010-kfo.pdf +qpdf: adding pages from 011-kfo.pdf +qpdf: adding pages from 012-kfo.pdf +qpdf: adding pages from 013-kfo.pdf +qpdf: adding pages from 014-kfo.pdf +qpdf: adding pages from 015-kfo.pdf +qpdf: adding pages from 016-kfo.pdf +qpdf: adding pages from 017-kfo.pdf +qpdf: adding pages from 018-kfo.pdf +qpdf: adding pages from 019-kfo.pdf qpdf: wrote file a.pdf