From 1873334054549b2c53b3aa2afd483d6bcda758aa Mon Sep 17 00:00:00 2001 From: Chris Sharp Date: Fri, 26 Jan 2018 07:58:58 -0500 Subject: [PATCH] LP#1745610 - Deprecate original hold_targeter.pl With the new hold targeter battle-tested for over a year, it's now time to promote it to default, deprecating the older version. This commit renames the old targeter to "hold_targeter_legacy.pl" and the new to "hold_targeter.pl", and adapts the example crontab to use the new targeter's parameter syntax. Signed-off-by: Chris Sharp Signed-off-by: Bill Erickson --- Open-ILS/examples/crontab.example | 2 +- Open-ILS/src/support-scripts/hold_targeter.pl | 270 ++++++++++++------ .../support-scripts/hold_targeter_legacy.pl | 99 +++++++ .../src/support-scripts/hold_targeter_v2.pl | 209 -------------- 4 files changed, 290 insertions(+), 290 deletions(-) create mode 100755 Open-ILS/src/support-scripts/hold_targeter_legacy.pl delete mode 100755 Open-ILS/src/support-scripts/hold_targeter_v2.pl diff --git a/Open-ILS/examples/crontab.example b/Open-ILS/examples/crontab.example index 8803b32f13..35144cd871 100644 --- a/Open-ILS/examples/crontab.example +++ b/Open-ILS/examples/crontab.example @@ -31,7 +31,7 @@ EG_BIN_DIR = /openils/bin # m h dom mon dow command # Run the hold targeter -*/15 * * * * . ~/.bashrc && $EG_BIN_DIR/hold_targeter.pl $SRF_CORE +*/15 * * * * . ~/.bashrc && $EG_BIN_DIR/hold_targeter.pl --osrf-config $SRF_CORE # Run the hold thawer 5 0 * * * . ~/.bashrc && $EG_BIN_DIR/thaw_expired_frozen_holds.srfsh diff --git a/Open-ILS/src/support-scripts/hold_targeter.pl b/Open-ILS/src/support-scripts/hold_targeter.pl index 2ca196e1ae..93eaf4d0f9 100755 --- a/Open-ILS/src/support-scripts/hold_targeter.pl +++ b/Open-ILS/src/support-scripts/hold_targeter.pl @@ -1,99 +1,209 @@ #!/usr/bin/perl -# --------------------------------------------------------------------- -# Usage: -# hold_targeter.pl -# --------------------------------------------------------------------- - use strict; use warnings; -use OpenSRF::Utils::JSON; +use Getopt::Long; use OpenSRF::System; +use OpenSRF::AppSession; use OpenSRF::Utils::SettingsClient; -use OpenSRF::MultiSession; -use OpenSRF::EX qw(:try); +use OpenILS::Utils::Fieldmapper; +$ENV{OSRF_LOG_CLIENT} = 1; +#---------------------------------------------------------------- +# Batch hold (re)targeter +# +# Usage: +# ./hold_targeter.pl /openils/conf/opensrf_core.xml +#---------------------------------------------------------------- + +my $help; +my $osrf_config = '/openils/conf/opensrf_core.xml'; +my $lockfile = '/tmp/hold_targeter-LOCK'; +my $parallel = 0; +my $verbose = 0; +my $retarget_interval; +my $soft_retarget_interval; +my $next_check_interval; +my $recv_timeout = 3600; +my $parallel_init_sleep = 0; + +# how often the server sends a summary reply per backend. +my $return_throttle = 500; + +GetOptions( + 'help' => \$help, + 'osrf-config=s' => \$osrf_config, + 'lockfile=s' => \$lockfile, + 'parallel=i' => \$parallel, + 'verbose' => \$verbose, + 'parallel-init-sleep=i' => \$parallel_init_sleep, + 'retarget-interval=s' => \$retarget_interval, + 'next-check-interval=s' => \$next_check_interval, + 'soft-retarget-interval=s' => \$soft_retarget_interval, +) || die "\nSee --help for more\n"; + +sub help { + print < + Number of parallel hold processors to run. This overrides any + value found in opensrf.xml + + --parallel-init-sleep + Number of seconds to wait before starting each subsequent + parallel targeter instance. This gives each targeter backend + time to run the large targetable holds query before the next + kicks off, so they don't all hit the database at once. + + Defaults to no sleep. + + --soft-retarget-interval + Holds whose previous check time sits between the + --soft-retarget-interval and the --retarget-interval are + treated like this: + + 1. The list of potential copies is updated for all matching holds. + 2. Holds that have a viable target are otherwise left untouched, + including their prev_check_time. + 3. Holds with no viable target are fully retargeted. + + --next-check-interval + Specify how long after the current run time the targeter will + retarget the currently affected holds. Applying a specific + interval is useful when the retarget_interval is shorter than + the time between targeter runs. + + This value is used to determine if an org unit will be closed + during the next iteration of the targeter. It overrides the + default behavior of calculating the next retarget time from the + retarget-interval. + + --retarget-interval + Retarget holds whose previous check time occured before the + requested interval. + Overrides the 'circ.holds.retarget_interval' global_flag value. + +HELP + + exit(0); } -open(F, ">$lockfile"); -print F $$; -close F; - -my $settings; -my $parallel; - -try { - OpenSRF::System->bootstrap_client( config_file => $config ); - $settings = OpenSRF::Utils::SettingsClient->new; - $parallel = $settings->config_value( hold_targeter => 'parallel' ) || 1; -} otherwise { - my $e = shift; - warn "$e\n"; - unlink $lockfile; - exit 1; -}; +help() if $help; + +sub init { -if ($parallel == 1) { - - try { - my $r = OpenSRF::AppSession - ->create( 'open-ils.storage' ) - ->request( 'open-ils.storage.action.hold_request.copy_targeter' => '24h' ); - - while (!$r->complete) { - my $start = time; - $r->recv(timeout => 3600); - last if (time() - $start) >= 3600; - }; - } otherwise { - my $e = shift; - warn "Failure in single-session targeter:\n$e\n"; - }; - -} else { - - try { - my $multi_targeter = OpenSRF::MultiSession->new( - app => 'open-ils.storage', - cap => $parallel, - api_level => 1, - session_hash_function => sub { - my $ses = shift; - my $req = shift; - return $_[-1]; # last parameter is the ID of the metarecord associated with the - # request's target; using this as the hash function value ensures - # that parallel targeters won't try to simultaneously handle two - # hold requests that have overlapping pools of copies that could - # fill those requests + OpenSRF::System->bootstrap_client(config_file => $osrf_config); + Fieldmapper->import( + IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL")); + + if (!$parallel) { + my $settings = OpenSRF::Utils::SettingsClient->new; + $parallel = $settings->config_value(hold_targeter => 'parallel') || 1; + } +} + +sub run_batches { + + # Hanging all of the parallel requests off the same app session + # lets us operate the same as a MultiSession batch with additional + # fine-grained controls over the receive timeout and real-time + # response handling. + my $ses = OpenSRF::AppSession->create('open-ils.hold-targeter'); + + my @reqs; + for my $slot (1..$parallel) { + + if ($slot > 1 && $parallel_init_sleep) { + $verbose && print "Sleeping $parallel_init_sleep ". + "seconds before targeter slot=$slot launch\n"; + sleep $parallel_init_sleep; + } + + $verbose && print "Starting targeter slot=$slot\n"; + + my $req = $ses->request( + 'open-ils.hold-targeter.target', { + return_count => 1, + return_throttle => $return_throttle, + parallel_count => $parallel, + parallel_slot => $slot, + retarget_interval => $retarget_interval, + next_check_interval => $next_check_interval, + soft_retarget_interval => $soft_retarget_interval } ); - - my $storage = OpenSRF::AppSession->create("open-ils.storage"); - - my $r = $storage->request('open-ils.storage.action.hold_request.targetable_holds.id_list', '24h'); - while ( my $h = $r->recv ) { - if ($r->failed) { - print $r->failed->stringify . "\n"; - last; - } - if (my $hold = $h->content) { - $multi_targeter->request( 'open-ils.storage.action.hold_request.copy_targeter', '', $hold->[0], $hold->[1]); + + $req->{_parallel_slot} = $slot; # for grouping/logging below + push(@reqs, $req); + } + + while (@reqs) { + my $start = time; + $ses->queue_wait($recv_timeout); # wait for a response + + # As a fail-safe, exit if no responses have arrived + # within the timeout interval. + last if (time - $start) >= $recv_timeout; + + for my $req (@reqs) { + # Pull all responses off the receive queues. + while (my $resp = $req->recv(0)) { + die $req->failed . "\n" if $req->failed; + + $verbose && print sprintf( + "Targeter [%d] processed %d holds\n", + $req->{_parallel_slot}, + $resp->content + ); } } - - $storage->disconnect(); - - $multi_targeter->session_wait(1); - $multi_targeter->disconnect; - } otherwise { - my $e = shift; - warn "Failure in multi-session targeter:\n$e\n"; + + @reqs = grep {!$_->complete} @reqs; } } +# ---- + +die "I seem to be running already. If not remove $lockfile, try again\n" + if -e $lockfile; + +open(LOCK, ">$lockfile") or die "Cannot open lock file: $lockfile : $@\n"; +print LOCK $$ or die "Cannot write to lock file: $lockfile : $@\n"; +close LOCK; + +eval { # Make sure we can delete the lock file. + + init(); + + my $start = time; + + run_batches(); + + my $minutes = sprintf('%0.2f', (time - $start) / 60.0); + + $verbose && print "Processing took $minutes minutes.\n"; +}; + +warn "Hold processing exited with error: $@\n" if $@; + unlink $lockfile; diff --git a/Open-ILS/src/support-scripts/hold_targeter_legacy.pl b/Open-ILS/src/support-scripts/hold_targeter_legacy.pl new file mode 100755 index 0000000000..2ca196e1ae --- /dev/null +++ b/Open-ILS/src/support-scripts/hold_targeter_legacy.pl @@ -0,0 +1,99 @@ +#!/usr/bin/perl +# --------------------------------------------------------------------- +# Usage: +# hold_targeter.pl +# --------------------------------------------------------------------- + +use strict; +use warnings; +use OpenSRF::Utils::JSON; +use OpenSRF::System; +use OpenSRF::Utils::SettingsClient; +use OpenSRF::MultiSession; +use OpenSRF::EX qw(:try); + +my $config = shift || die "bootstrap config required\n"; +my $lockfile = shift || "/tmp/hold_targeter-LOCK"; + +if (-e $lockfile) { + die "I seem to be running already. If not remove $lockfile, try again\n"; +} + +open(F, ">$lockfile"); +print F $$; +close F; + +my $settings; +my $parallel; + +try { + OpenSRF::System->bootstrap_client( config_file => $config ); + $settings = OpenSRF::Utils::SettingsClient->new; + $parallel = $settings->config_value( hold_targeter => 'parallel' ) || 1; +} otherwise { + my $e = shift; + warn "$e\n"; + unlink $lockfile; + exit 1; +}; + +if ($parallel == 1) { + + try { + my $r = OpenSRF::AppSession + ->create( 'open-ils.storage' ) + ->request( 'open-ils.storage.action.hold_request.copy_targeter' => '24h' ); + + while (!$r->complete) { + my $start = time; + $r->recv(timeout => 3600); + last if (time() - $start) >= 3600; + }; + } otherwise { + my $e = shift; + warn "Failure in single-session targeter:\n$e\n"; + }; + +} else { + + try { + my $multi_targeter = OpenSRF::MultiSession->new( + app => 'open-ils.storage', + cap => $parallel, + api_level => 1, + session_hash_function => sub { + my $ses = shift; + my $req = shift; + return $_[-1]; # last parameter is the ID of the metarecord associated with the + # request's target; using this as the hash function value ensures + # that parallel targeters won't try to simultaneously handle two + # hold requests that have overlapping pools of copies that could + # fill those requests + } + ); + + my $storage = OpenSRF::AppSession->create("open-ils.storage"); + + my $r = $storage->request('open-ils.storage.action.hold_request.targetable_holds.id_list', '24h'); + while ( my $h = $r->recv ) { + if ($r->failed) { + print $r->failed->stringify . "\n"; + last; + } + if (my $hold = $h->content) { + $multi_targeter->request( 'open-ils.storage.action.hold_request.copy_targeter', '', $hold->[0], $hold->[1]); + } + } + + $storage->disconnect(); + + $multi_targeter->session_wait(1); + $multi_targeter->disconnect; + } otherwise { + my $e = shift; + warn "Failure in multi-session targeter:\n$e\n"; + } +} + +unlink $lockfile; + diff --git a/Open-ILS/src/support-scripts/hold_targeter_v2.pl b/Open-ILS/src/support-scripts/hold_targeter_v2.pl deleted file mode 100755 index 93eaf4d0f9..0000000000 --- a/Open-ILS/src/support-scripts/hold_targeter_v2.pl +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; -use Getopt::Long; -use OpenSRF::System; -use OpenSRF::AppSession; -use OpenSRF::Utils::SettingsClient; -use OpenILS::Utils::Fieldmapper; -$ENV{OSRF_LOG_CLIENT} = 1; -#---------------------------------------------------------------- -# Batch hold (re)targeter -# -# Usage: -# ./hold_targeter.pl /openils/conf/opensrf_core.xml -#---------------------------------------------------------------- - -my $help; -my $osrf_config = '/openils/conf/opensrf_core.xml'; -my $lockfile = '/tmp/hold_targeter-LOCK'; -my $parallel = 0; -my $verbose = 0; -my $retarget_interval; -my $soft_retarget_interval; -my $next_check_interval; -my $recv_timeout = 3600; -my $parallel_init_sleep = 0; - -# how often the server sends a summary reply per backend. -my $return_throttle = 500; - -GetOptions( - 'help' => \$help, - 'osrf-config=s' => \$osrf_config, - 'lockfile=s' => \$lockfile, - 'parallel=i' => \$parallel, - 'verbose' => \$verbose, - 'parallel-init-sleep=i' => \$parallel_init_sleep, - 'retarget-interval=s' => \$retarget_interval, - 'next-check-interval=s' => \$next_check_interval, - 'soft-retarget-interval=s' => \$soft_retarget_interval, -) || die "\nSee --help for more\n"; - -sub help { - print < - Number of parallel hold processors to run. This overrides any - value found in opensrf.xml - - --parallel-init-sleep - Number of seconds to wait before starting each subsequent - parallel targeter instance. This gives each targeter backend - time to run the large targetable holds query before the next - kicks off, so they don't all hit the database at once. - - Defaults to no sleep. - - --soft-retarget-interval - Holds whose previous check time sits between the - --soft-retarget-interval and the --retarget-interval are - treated like this: - - 1. The list of potential copies is updated for all matching holds. - 2. Holds that have a viable target are otherwise left untouched, - including their prev_check_time. - 3. Holds with no viable target are fully retargeted. - - --next-check-interval - Specify how long after the current run time the targeter will - retarget the currently affected holds. Applying a specific - interval is useful when the retarget_interval is shorter than - the time between targeter runs. - - This value is used to determine if an org unit will be closed - during the next iteration of the targeter. It overrides the - default behavior of calculating the next retarget time from the - retarget-interval. - - --retarget-interval - Retarget holds whose previous check time occured before the - requested interval. - Overrides the 'circ.holds.retarget_interval' global_flag value. - -HELP - - exit(0); -} - -help() if $help; - -sub init { - - OpenSRF::System->bootstrap_client(config_file => $osrf_config); - Fieldmapper->import( - IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL")); - - if (!$parallel) { - my $settings = OpenSRF::Utils::SettingsClient->new; - $parallel = $settings->config_value(hold_targeter => 'parallel') || 1; - } -} - -sub run_batches { - - # Hanging all of the parallel requests off the same app session - # lets us operate the same as a MultiSession batch with additional - # fine-grained controls over the receive timeout and real-time - # response handling. - my $ses = OpenSRF::AppSession->create('open-ils.hold-targeter'); - - my @reqs; - for my $slot (1..$parallel) { - - if ($slot > 1 && $parallel_init_sleep) { - $verbose && print "Sleeping $parallel_init_sleep ". - "seconds before targeter slot=$slot launch\n"; - sleep $parallel_init_sleep; - } - - $verbose && print "Starting targeter slot=$slot\n"; - - my $req = $ses->request( - 'open-ils.hold-targeter.target', { - return_count => 1, - return_throttle => $return_throttle, - parallel_count => $parallel, - parallel_slot => $slot, - retarget_interval => $retarget_interval, - next_check_interval => $next_check_interval, - soft_retarget_interval => $soft_retarget_interval - } - ); - - $req->{_parallel_slot} = $slot; # for grouping/logging below - push(@reqs, $req); - } - - while (@reqs) { - my $start = time; - $ses->queue_wait($recv_timeout); # wait for a response - - # As a fail-safe, exit if no responses have arrived - # within the timeout interval. - last if (time - $start) >= $recv_timeout; - - for my $req (@reqs) { - # Pull all responses off the receive queues. - while (my $resp = $req->recv(0)) { - die $req->failed . "\n" if $req->failed; - - $verbose && print sprintf( - "Targeter [%d] processed %d holds\n", - $req->{_parallel_slot}, - $resp->content - ); - } - } - - @reqs = grep {!$_->complete} @reqs; - } -} - -# ---- - -die "I seem to be running already. If not remove $lockfile, try again\n" - if -e $lockfile; - -open(LOCK, ">$lockfile") or die "Cannot open lock file: $lockfile : $@\n"; -print LOCK $$ or die "Cannot write to lock file: $lockfile : $@\n"; -close LOCK; - -eval { # Make sure we can delete the lock file. - - init(); - - my $start = time; - - run_batches(); - - my $minutes = sprintf('%0.2f', (time - $start) / 60.0); - - $verbose && print "Processing took $minutes minutes.\n"; -}; - -warn "Hold processing exited with error: $@\n" if $@; - -unlink $lockfile; - -- 2.43.2