Open-ILS/src/extras/import/direct_ingest.pl

   1 #!/usr/bin/perl
   2 use strict;
   3 use warnings;
   4
   5 use OpenSRF::System;
   6 use OpenSRF::EX qw/:try/;
   7 use OpenSRF::AppSession;
   8 use OpenSRF::Application;
   9 use OpenSRF::MultiSession;
  10 use OpenSRF::Utils::SettingsClient;
  11 use OpenILS::Application::Ingest;
  12 use OpenILS::Application::AppUtils;
  13 use OpenILS::Utils::Fieldmapper;
  14 use Digest::MD5 qw/md5_hex/;
  15 use OpenSRF::Utils::JSON;
  16 use Data::Dumper;
  17 use FileHandle;
  18
  19 use Time::HiRes qw/time/;
  20 use Getopt::Long;
  21 use MARC::Batch;
  22 use MARC::File::XML (BinaryEncoding => 'UTF-8');
  23 use MARC::Charset;
  24
  25 MARC::Charset->ignore_errors(1);
  26
  27 my ($max_uri, $max_cn, $auth, $config, $quiet) =
  28         (0, 0, 0, '/openils/conf/opensrf_core.xml');
  29
  30 GetOptions(
  31         'config=s'      => \$config,
  32         'authority'     => \$auth,
  33         'quiet'         => \$quiet,
  34         'max_uri=i'     => \$max_uri,
  35         'max_cn=i'      => \$max_cn,
  36 );
  37
  38 my @ses;
  39
  40 open NEWERR,     ">&STDERR";
  41
  42 select NEWERR; $| = 1;
  43 select STDERR; $| = 1;
  44 select STDOUT; $| = 1;
  45
  46 OpenSRF::System->bootstrap_client( config_file => $config );
  47 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
  48
  49 OpenILS::Application::Ingest->use;
  50
  51 die "We have no more use for authority or biblio ingest ... just insert the are or bre objects and you're done!\n";
  52
  53 my $meth = 'open-ils.ingest.full.biblio.object.readonly';
  54 $meth = 'open-ils.ingest.full.authority.object.readonly' if ($auth);
  55
  56 $meth = OpenILS::Application::Ingest->method_lookup( $meth );
  57
  58 my $count = 0;
  59 my $starttime = time;
  60 while (my $rec = <>) {
  61         next unless ($rec);
  62
  63         my $bib = OpenSRF::Utils::JSON->JSON2perl($rec);
  64         my $data;
  65
  66         try {
  67                 ($data) = $meth->run( $bib => $max_cn => $max_uri );
  68         } catch Error with {
  69                 my $e = shift;
  70                 warn "Couldn't process record: $e\n >>> $rec\n";
  71         };
  72
  73         next unless $data;
  74
  75         postprocess( { bib => $bib, ingest_data => $data } );
  76
  77         if (!$quiet && !($count % 20)) {
  78                 print NEWERR "\r$count\t". $count / (time - $starttime);
  79         }
  80
  81         $count++;
  82 }
  83
  84 sub postprocess {
  85         my $data = shift;
  86
  87         my ($field_entries, $fp, $rd, $uri);
  88
  89         my $bib = $data->{bib};
  90         my $full_rec = $data->{ingest_data}->{full_rec};
  91
  92         if (!$auth) {
  93                 $field_entries = $data->{ingest_data}->{field_entries};
  94                 $fp = $data->{ingest_data}->{fingerprint};
  95                 $rd = $data->{ingest_data}->{descriptor};
  96                 $uri = $data->{ingest_data}->{uri};
  97
  98                 $bib->fingerprint( $fp->{fingerprint} );
  99                 $bib->quality( $fp->{quality} );
 100         }
 101
 102         print( OpenSRF::Utils::JSON->perl2JSON($bib)."\n" );
 103         if (!$auth) {
 104                 print( OpenSRF::Utils::JSON->perl2JSON($rd)."\n" );
 105                 print( OpenSRF::Utils::JSON->perl2JSON($_)."\n" ) for (@$field_entries);
 106                 for my $u (@$uri) {
 107                         print( OpenSRF::Utils::JSON->perl2JSON($u->{call_number})."\n" ) if $u->{call_number}->isnew;
 108                         print( OpenSRF::Utils::JSON->perl2JSON($u->{uri})."\n" ) if $u->{uri}->isnew;
 109
 110                         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 111                         $umap->uri($u->{uri}->id);
 112                         $umap->call_number($u->{call_number}->id);
 113                         print( OpenSRF::Utils::JSON->perl2JSON($umap)."\n" );
 114
 115                         $max_cn = $u->{call_number}->id + 1 if $u->{call_number}->isnew;
 116                         $max_uri = $u->{uri}->id + 1 if $u->{uri}->isnew;
 117                 }
 118         }
 119
 120         print( OpenSRF::Utils::JSON->perl2JSON($_)."\n" ) for (@$full_rec);
 121 }
 122