From a8d100b1b56ebbf377a76e23dd9e3710e3ad38c8 Mon Sep 17 00:00:00 2001 From: miker Date: Fri, 20 Apr 2007 14:01:42 +0000 Subject: [PATCH] changing the holding retrieval strategy git-svn-id: svn://svn.open-ils.org/ILS/trunk@7154 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/src/support-scripts/marc_export | 177 ++++++++++++++++------- 1 file changed, 125 insertions(+), 52 deletions(-) diff --git a/Open-ILS/src/support-scripts/marc_export b/Open-ILS/src/support-scripts/marc_export index 6dc7bed45c..09775e63d0 100755 --- a/Open-ILS/src/support-scripts/marc_export +++ b/Open-ILS/src/support-scripts/marc_export @@ -14,10 +14,11 @@ use MARC::Record; use MARC::File::XML; use UNIVERSAL::require; +use Time::HiRes qw/time/; use Getopt::Long; -my @formats = qw/USMARC UNIMARC XML/; +my @formats = qw/USMARC UNIMARC XML BRE/; my ($config,$format,$encoding,$location,$dollarsign,$idl,$help,$holdings) = ('/openils/conf/bootstrap.conf','USMARC','MARC8','','$'); @@ -58,7 +59,7 @@ $encoding = uc($encoding); binmode(STDOUT, ':raw') if ($encoding ne 'UTF-8'); binmode(STDOUT, ':utf8') if ($encoding eq 'UTF-8'); -if (!grep { $format eq $_ } @formats) { +if (!grep { uc($format) eq $_ } @formats) { die "Please select a supported format. ". "Right now that means one of [". join('|',@formats). "]\n"; @@ -86,22 +87,59 @@ HEADER my %orgs; my %shelves; -if ($holdings) { - my $o = $ses->request( 'open-ils.cstore.direct.actor.org_unit.search.atomic', { id => { '!=' => undef } } )->gather(1); - %orgs = map { ( $_->id => $_ ) } @$o; - my $s = $ses->request( 'open-ils.cstore.direct.asset.copy_location.search.atomic', { id => { '!=' => undef } } )->gather(1); - %shelves = map { ( $_->id => $_ ) } @$s; +my $flesh = {}; +if ($holdings) { + my $r = $ses->request( 'open-ils.cstore.direct.actor.org_unit.search', { id => { '!=' => undef } } ); + + while (my $o = $r->recv) { + die $r->failed->stringify if ($r->failed); + $o = $o->content; + last unless ($o); + $orgs{$o->id} = $o; + } + $r->finish; + + $r = $ses->request( 'open-ils.cstore.direct.asset.copy_location.search', { id => { '!=' => undef } } ); + + while (my $s = $r->recv) { + die $r->failed->stringify if ($r->failed); + $s = $s->content; + last unless ($s); + $shelves{$s->id} = $s; + } + $r->finish; + + $flesh = { flesh => 1, flesh_fields => { bre => [ 'call_numbers' ], acn => [ 'copies' ] } }; } my $start = time; -my $count = 1; +my $last_time = time; +my %count = (); +my $speed = 0; while ( my $i = <> ) { - my $bib = $ses->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve', $i )->gather(1); - + my $bib; + try { + local $SIG{ALRM} = sub { die "TIMEOUT\n" }; + alarm(1); + $bib = $ses->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve', $i, $flesh )->gather(1); + alarm(0); + } otherwise { + warn "\n!!!!!! Timed out trying to read record $i\n"; + }; + alarm(0); + + $count{bib}++; next unless $bib; + if (uc($format) eq 'BRE') { + print JSON->perl2JSON($bib); + stats(); + next; + } + try { + my $r = MARC::Record->new_from_xml( $bib->marc, $encoding, $format ); $r->delete_field( $_ ) for ($r->field(901)); @@ -113,45 +151,48 @@ while ( my $i = <> ) { ) ); - if ($holdings) { - my $cn_list = $ses->request( - 'open-ils.cstore.direct.asset.call_number.search.atomic', - { record => $i, deleted => 'f' } - )->gather(1); - - - my $cp_list = $ses->request( - 'open-ils.cstore.direct.asset.copy.search.atomic', - { call_number => [ map { $_->id } @$cn_list ], deleted => 'f' } - )->gather(1); - - my %cn_map; - push @{$cn_map{$_->call_number}}, $_ for (@$cp_list); - - for my $cn ( @$cn_list ) { - my $cn_map_list = $cn_map{$cn->id}; - for my $cp ( @$cn_map_list ) { - $r->append_fields( - MARC::Field->new( - 852, '4', '', - a => $location, - b => $orgs{$cn->owning_lib}->shortname, - b => $orgs{$cp->circ_lib}->shortname, - c => $shelves{$cp->location}->name, - j => $cn->label, - ($cp->circ_modifier ? ( g => $cp->circ_modifier ) : ()), - p => $cp->barcode, - ($cp->price ? ( y => $dollarsign.$cp->price ) : ()), - ($cp->copy_number ? ( t => $cp->copy_number ) : ()), - ($cp->ref eq 't' ? ( x => 'reference' ) : ()), - ($cp->holdable eq 'f' ? ( x => 'unholdable' ) : ()), - ($cp->circulate eq 'f' ? ( x => 'noncirculating' ) : ()), - ($cp->opac_visible eq 'f' ? ( x => 'hidden' ) : ()), - ) - ); + + my $cn_list = $bib->call_numbers; + if ($cn_list && @$cn_list) { + + $count{cn} += @$cn_list; + + my $cp_list = [ map { @{ $_->copies } } @$cn_list ]; + if ($cp_list && @$cp_list) { + + my %cn_map; + push @{$cn_map{$_->call_number}}, $_ for (@$cp_list); + + for my $cn ( @$cn_list ) { + my $cn_map_list = $cn_map{$cn->id}; + + for my $cp ( @$cn_map_list ) { + $count{cp}++; + + $r->append_fields( + MARC::Field->new( + 852, '4', '', + a => $location, + b => $orgs{$cn->owning_lib}->shortname, + b => $orgs{$cp->circ_lib}->shortname, + c => $shelves{$cp->location}->name, + j => $cn->label, + ($cp->circ_modifier ? ( g => $cp->circ_modifier ) : ()), + p => $cp->barcode, + ($cp->price ? ( y => $dollarsign.$cp->price ) : ()), + ($cp->copy_number ? ( t => $cp->copy_number ) : ()), + ($cp->ref eq 't' ? ( x => 'reference' ) : ()), + ($cp->holdable eq 'f' ? ( x => 'unholdable' ) : ()), + ($cp->circulate eq 'f' ? ( x => 'noncirculating' ) : ()), + ($cp->opac_visible eq 'f' ? ( x => 'hidden' ) : ()), + ) + ); + + stats() if (! ($count{cp} % 100 )); + } } } - } + } if (uc($format) eq 'XML') { print $r->as_xml_record; @@ -160,16 +201,48 @@ while ( my $i = <> ) { } elsif (uc($format) eq 'USMARC') { print $r->as_usmarc } - } catch { + + $count{did}++; + + } otherwise { my $e = shift; warn "\n$e\n"; }; - if (! ($count % 50 )) { - my $speed = sprintf( '%0.4f', $count / time - $start ); - print STDERR "\r $speed / second \r"; - } + stats() if (! ($count{bib} % 50 )); } print "\n" if ($format eq 'XML'); +$speed = $count{did} / (time - $start); +my $time = time - $start; +print STDERR <