Open-ILS/src/support-scripts/marc_export

   1 #!/usr/bin/perl
   2 use strict;
   3 use warnings;
   4 use bytes;
   5
   6 use OpenSRF::System;
   7 use OpenSRF::EX qw/:try/;
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenILS::Application::AppUtils;
  11 use OpenILS::Utils::Fieldmapper;
  12
  13 use MARC::Record;
  14 use MARC::File::XML;
  15 use UNIVERSAL::require;
  16
  17 use Time::HiRes qw/time/;
  18 use Getopt::Long;
  19
  20
  21 my @formats = qw/USMARC UNIMARC XML BRE/;
  22
  23 my ($config,$format,$encoding,$location,$dollarsign,$idl,$help,$holdings) = ('/openils/conf/bootstrap.conf','USMARC','MARC8','','$');
  24
  25 GetOptions(
  26         'help'      => \$help,
  27         'items'      => \$holdings,
  28         'location=s'      => \$location,
  29         'money=s'      => \$dollarsign,
  30         'config=s'      => \$config,
  31         'format=s'      => \$format,
  32         'xml-idl=s'      => \$idl,
  33         'encoding=s'      => \$encoding,
  34 );
  35
  36 if ($help) {
  37         print <<"       HELP";
  38 Usage: $0 [options]
  39  --help or -h           This screen.
  40  --config or -c         Configuration file [/openils/conf/bootstrap.conf]
  41  --format or -f         Output format (USMARC, UNIMARC, XML) [USMARC]
  42  --encoding or -e       Output Encoding (UTF-8, ISO-8859-?, MARC8) [MARC8]
  43  --items or -i          Include items (holdings) in the output
  44  --xml-idl or -x        Location of the IDL XML
  45  --location or -l       MARC Location Code for holdings from
  46                         http://www.loc.gov/marc/organizations/orgshome.html
  47
  48 Example:
  49
  50   cat list_of_ids | $0 > output_file
  51
  52         HELP
  53         exit;
  54 }
  55
  56 $format = uc($format);
  57 $encoding = uc($encoding);
  58
  59 binmode(STDOUT, ':raw') if ($encoding ne 'UTF-8');
  60 binmode(STDOUT, ':utf8') if ($encoding eq 'UTF-8');
  61
  62 if (!grep { uc($format) eq $_ } @formats) {
  63         die     "Please select a supported format.  ".
  64                 "Right now that means one of [".
  65                 join('|',@formats). "]\n";
  66 }
  67
  68 if ($format ne 'XML') {
  69         my $type = 'MARC::File::' . $format;
  70         $type->require;
  71 }
  72
  73 OpenSRF::System->bootstrap_client( config_file => $config );
  74
  75 if (!$idl) {
  76         $idl = OpenSRF::Utils::SettingsClient->new->config_value("IDL");
  77 }
  78
  79 Fieldmapper->import(IDL => $idl);
  80
  81 my $ses = OpenSRF::AppSession->create('open-ils.cstore');
  82
  83 print <<HEADER if ($format eq 'XML');
  84 <?xml version="1.0" encoding="$encoding"?>
  85 <collection xmlns='http://www.loc.gov/MARC21/slim'>
  86 HEADER
  87
  88 my %orgs;
  89 my %shelves;
  90
  91 my $flesh = {};
  92 if ($holdings) {
  93         my $r = $ses->request( 'open-ils.cstore.direct.actor.org_unit.search', { id => { '!=' => undef } } );
  94
  95     while (my $o = $r->recv) {
  96         die $r->failed->stringify if ($r->failed);
  97         $o = $o->content;
  98         last unless ($o);
  99             $orgs{$o->id} = $o;
 100     }
 101     $r->finish;
 102
 103         $r = $ses->request( 'open-ils.cstore.direct.asset.copy_location.search', { id => { '!=' => undef } } );
 104
 105     while (my $s = $r->recv) {
 106         die $r->failed->stringify if ($r->failed);
 107         $s = $s->content;
 108         last unless ($s);
 109             $shelves{$s->id} = $s;
 110     }
 111     $r->finish;
 112
 113     $flesh = { flesh => 1, flesh_fields => { bre => [ 'call_numbers' ], acn => [ 'copies' ] } };
 114 }
 115
 116 my $start = time;
 117 my $last_time = time;
 118 my %count = ();
 119 my $speed = 0;
 120 while ( my $i = <> ) {
 121     my $bib;
 122     try {
 123         local $SIG{ALRM} = sub { die "TIMEOUT\n" };
 124         alarm(1);
 125             $bib = $ses->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve', $i, $flesh )->gather(1);
 126         alarm(0);
 127     } otherwise {
 128         warn "\n!!!!!! Timed out trying to read record $i\n";
 129     };
 130     alarm(0);
 131
 132     $count{bib}++;
 133         next unless $bib;
 134
 135     if (uc($format) eq 'BRE') {
 136         print JSON->perl2JSON($bib);
 137             stats();
 138         next;
 139     }
 140
 141         try {
 142
 143                 my $r = MARC::Record->new_from_xml( $bib->marc, $encoding, $format );
 144                 $r->delete_field( $_ ) for ($r->field(901));
 145
 146                 $r->append_fields(
 147                         MARC::Field->new(
 148                                 901, '', '',
 149                                 a => $bib->tcn_value,
 150                                 b => $bib->tcn_source
 151                         )
 152                 );
 153
 154
 155         my $cn_list = $bib->call_numbers;
 156         if ($cn_list && @$cn_list) {
 157
 158                 $count{cn} += @$cn_list;
 159
 160             my $cp_list = [ map { @{ $_->copies } } @$cn_list ];
 161             if ($cp_list && @$cp_list) {
 162
 163                     my %cn_map;
 164                     push @{$cn_map{$_->call_number}}, $_ for (@$cp_list);
 165
 166                     for my $cn ( @$cn_list ) {
 167                         my $cn_map_list = $cn_map{$cn->id};
 168
 169                         for my $cp ( @$cn_map_list ) {
 170                             $count{cp}++;
 171
 172                                                 $r->append_fields(
 173                                                         MARC::Field->new(
 174                                                                 852, '4', '',
 175                                                                 a => $location,
 176                                                                 b => $orgs{$cn->owning_lib}->shortname,
 177                                                                 b => $orgs{$cp->circ_lib}->shortname,
 178                                                                 c => $shelves{$cp->location}->name,
 179                                                                 j => $cn->label,
 180                                                                 ($cp->circ_modifier ? ( g => $cp->circ_modifier ) : ()),
 181                                                                 p => $cp->barcode,
 182                                                                 ($cp->price ? ( y => $dollarsign.$cp->price ) : ()),
 183                                                                 ($cp->copy_number ? ( t => $cp->copy_number ) : ()),
 184                                                                 ($cp->ref eq 't' ? ( x => 'reference' ) : ()),
 185                                                                 ($cp->holdable eq 'f' ? ( x => 'unholdable' ) : ()),
 186                                                                 ($cp->circulate eq 'f' ? ( x => 'noncirculating' ) : ()),
 187                                                                 ($cp->opac_visible eq 'f' ? ( x => 'hidden' ) : ()),
 188                                                         )
 189                                                 );
 190
 191                         stats() if (! ($count{cp} % 100 ));
 192                                         }
 193                                 }
 194                         }
 195         }
 196
 197                 if (uc($format) eq 'XML') {
 198                         print $r->as_xml_record;
 199                 } elsif (uc($format) eq 'UNIMARC') {
 200                         print $r->as_unimarc
 201                 } elsif (uc($format) eq 'USMARC') {
 202                         print $r->as_usmarc
 203                 }
 204
 205         $count{did}++;
 206
 207         } otherwise {
 208                 my $e = shift;
 209                 warn "\n$e\n";
 210         };
 211
 212         stats() if (! ($count{bib} % 50 ));
 213 }
 214
 215 print "</collection>\n" if ($format eq 'XML');
 216
 217 $speed = $count{did} / (time - $start);
 218 my $time = time - $start;
 219 print STDERR <<DONE;
 220
 221 Exports Attempted : $count{bib}
 222 Exports Completed : $count{did}
 223 Overall Speed     : $speed
 224 Total Time Elapsed: $time seconds
 225
 226 DONE
 227
 228
 229 sub stats {
 230     try {
 231         $speed = $count{did} / (time - $start);
 232
 233         my $speed_now = ($count{did} - $count{did_last}) / (time - $count{time_last});
 234             my $cn_speed = $count{cn} / (time - $start);
 235         my $cp_speed = $count{cp} / (time - $start);
 236
 237             printf STDERR "\r  $count{did} of $count{bib} @  \%0.4f/s ttl / \%0.4f/s rt ".
 238                 "($count{cn} CNs @ \%0.4f/s :: $count{cp} CPs @ \%0.4f/s)\r",
 239                 $speed,
 240                 $speed_now,
 241                 $cn_speed,
 242                 $cp_speed;
 243     } otherwise {};
 244         $count{did_last} = $count{did};
 245         $count{time_last} = time;
 246 }
 247
 248