5 use lib '/openils/lib/perl5/';
8 use OpenSRF::EX qw/:try/;
9 use OpenSRF::AppSession;
10 use OpenSRF::Application;
11 use OpenSRF::MultiSession;
12 use OpenSRF::Utils::SettingsClient;
13 use OpenILS::Application::Ingest;
14 use OpenILS::Application::AppUtils;
15 use OpenILS::Utils::Fieldmapper;
16 use Digest::MD5 qw/md5_hex/;
21 use Time::HiRes qw/time/;
27 MARC::Charset->ignore_errors(1);
29 my ($workers, $config, $prefix) =
30 (1, '/openils/conf/bootstrap.conf', 'marc-out-');
33 'threads=i' => \$workers,
34 'config=s' => \$config,
35 'prefix=s' => \$prefix,
40 open NEWERR, ">&STDERR";
42 select NEWERR; $| = 1;
43 select STDERR; $| = 1;
44 select STDOUT; $| = 1;
52 $0 = "Local Ingest Worker $_";
61 $0 = "Local Ingest Master";
67 OpenSRF::System->bootstrap_client( config_file => $config );
68 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
72 OpenILS::Application::Ingest->use;
74 my $fname = "${prefix}$file";
79 my $f = new FileHandle(">$fname");
81 while (my $rec = <$pipe>) {
83 my $bib = JSON->JSON2perl($rec);
87 ($data) = OpenILS::Application::Ingest
88 ->method_lookup( 'open-ils.ingest.full.biblio.object.readonly' )
92 warn "Couldn't process record: $e\n >>> $rec\n";
107 my $starttime = time;
108 while ( my $rec = <> ) {
110 my $session_index = $count % $workers;
112 $ses[$session_index]->printflush( $rec );
114 if (!($count % 20)) {
115 print NEWERR "\r$count\t". $count / (time - $starttime);
121 $ses[$_]->close for (@ses);
126 my $bib = $data->{bib};
127 my $field_entries = $data->{worm_data}->{field_entries};
128 my $full_rec = $data->{worm_data}->{full_rec};
129 my $fp = $data->{worm_data}->{fingerprint};
130 my $rd = $data->{worm_data}->{descriptor};
132 $bib->fingerprint( $fp->{fingerprint} );
133 $bib->quality( $fp->{quality} );
135 $f->printflush( JSON->perl2JSON($bib)."\n" );
136 $f->printflush( JSON->perl2JSON($rd)."\n" );
137 $f->printflush( JSON->perl2JSON($_)."\n" ) for (@$field_entries);
138 $f->printflush( JSON->perl2JSON($_)."\n" ) for (@$full_rec);