2 # Copyright (C) 2008 Equinox Software, Inc.
3 # Author: Bill Erickson <erickson@esilibrary.com>
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # of the License, or (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
16 use strict; use warnings;
17 use Net::Server::PreFork;
18 use base qw/Net::Server::PreFork/;
22 use MARC::File::USMARC;
25 use File::Basename qw/fileparse/;
26 use Getopt::Long qw(:DEFAULT GetOptionsFromArray);
29 use OpenSRF::Utils::Logger qw/$logger/;
30 use OpenILS::Utils::Cronscript;
31 require 'oils_header.pl';
32 use vars qw/$apputils/;
40 # 'osrf-config=s' => '/openils/conf/opensrf_core.xml',
50 $OpenILS::Utils::Cronscript::debug=1 if $debug;
51 $Getopt::Long::debug=1 if $debug > 1;
52 my $o = OpenILS::Utils::Cronscript->new(\%defaults);
56 if (grep {$_ eq '--'} @ARGV) {
57 print "Splitting options into groups\n" if $debug;
60 $_ eq '--' and last; # stop at the first --
61 push @script_args, $_;
67 print "Calling MyGetOptions ",
68 (@script_args ? "with options: " . join(' ', @script_args) : 'without options from command line'),
71 my $real_opts = $o->MyGetOptions(\@script_args);
73 # GetOptionsFromArray(\@script_args, \%defaults, %defaults); # similar to
75 my $bufsize = $real_opts->{buffsize};
76 my $bib_source = $real_opts->{source};
77 my $osrf_config = $real_opts->{'osrf-config'};
78 my $oils_username = $real_opts->{user};
79 my $oils_password = $real_opts->{password};
80 my $help = $real_opts->{help};
81 my $merge_profile = $real_opts->{merge_profile};
82 my $queue_id = $real_opts->{queue};
83 my $tempdir = $real_opts->{tempdir} || tempdir();
84 $debug += $real_opts->{debug};
86 foreach (keys %$real_opts) {
87 print("real_opt->{$_} = ", $real_opts->{$_}, "\n") if $real_opts->{debug} or $debug;
89 my $wait_time = $real_opts->{wait};
92 # DEFAULTS for Net::Server
93 my $filename = fileparse($0, '.pl');
94 my $conf_file = (-r "$filename.conf") ? "$filename.conf" : undef;
95 # $conf_file is the Net::Server config for THIS script (not EG), if it exists and is readable
100 pod2usage(1) if $help;
101 unless ($oils_password) {
102 print STDERR "\nERROR: password option required for session login\n\n";
106 print Dumper($o) if $debug;
109 foreach my $ref (qw/bufsize bib_source osrf_config oils_username oils_password help conf_file debug/) {
111 printf "%16s => %s\n", $ref, (eval("\$$ref") || '');
116 print Dumper($real_opts);
121 return $apputils->simplereq( qw# opensrf.settings opensrf.settings.xpath.get
122 /opensrf/default/apps/open-ils.vandelay/app_settings/databases/importer # ) || '/tmp/foobar/';
128 WARNING: This script provides no security layer. Any client that has
129 access to the server+port can inject MARC records into the system.
135 return $apputils->simplereq(
137 'open-ils.cat.biblio.record.xml.import',
142 sub old_process_batch_data {
143 my $data = shift or $logger->error("process_batch_data called without any data");
147 open $handle, '<', \$data;
148 my $batch = MARC::Batch->new('USMARC', $handle);
156 eval { $rec = $batch->next; };
159 $logger->error("Failed parsing MARC record $index");
162 last unless $rec; # The only way out
164 my $resp = xml_import($authtoken, $rec->as_xml_record, $bib_source);
166 # has the session timed out?
167 if (oils_event_equals($resp, 'NO_SESSION')) {
169 $resp = xml_import($authtoken, $rec->as_xml_record, $bib_source); # try again w/ new token
171 oils_event_die($resp);
176 sub process_spool { # (authtoken, queue_id, filename, bib_source)
177 $apputils->simplereq('open-ils.vandelay', 'open-ils.vandelay.bib.process_spool', $authtoken, undef,
178 $queue_id, 'import', $filename, $bib_source );
180 sub bib_queue_import {
181 my $extra = {auto_overlay_exact => 1};
182 $extra->{merge_profile} = $merge_profile if $merge_profile;
183 $apputils->simplereq('open-ils.vandelay', 'open-ils.vandelay.bib_queue.import', $authtoken,
187 sub process_batch_data {
188 my $data = shift or $logger->error("process_batch_data called without any data");
191 my $resp = process_spool();
193 if (oils_event_equals($resp, 'NO_SESSION')) { # has the session timed out?
195 $resp = process_spool(); # try again w/ new token
198 $resp = bib_queue_import();
200 if (oils_event_equals($resp, 'NO_SESSION')) { # has the session timed out?
202 $resp = bib_queue_import(); # try again w/ new token
204 oils_event_die($resp);
207 sub process_request { # The core Net::Server method
209 my $socket = $self->{server}->{client};
213 # Reading <STDIN> blocks until the client is closed. Instead of waiting
214 # for that, give each inbound record $wait_time seconds to fully arrive
215 # and pull the data directly from the socket
217 local $SIG{ALRM} = sub { die "alarm\n" };
220 last unless $socket->sysread($buf, $bufsize);
225 if ($real_opts->{noqueue}) {
226 old_process_batch_data($data);
228 process_batch_data($data);
233 # the authtoken will timeout after the configured inactivity period.
234 # When that happens, get a new one.
236 $authtoken = oils_login($oils_username, $oils_password, 'staff')
237 or die "Unable to login to Evergreen as user $oils_username";
243 osrf_connect($osrf_config);
245 print "Calling Net::Server run ", (@ARGV ? "with command-line options: " . join(' ', @ARGV) : ''), "\n";
246 __PACKAGE__->run(conf_file => $conf_file);
252 marc_stream_importer.pl - Import MARC records via bare socket connection.
256 ./marc_stream_importer.pl [common opts ...] [script opts ...] -- [Net::Server opts ...] &
258 This script uses the EG common options from B<Cronscript>. See --help output for those.
260 Run C<perldoc marc_stream_importer.pl> for full documentation.
262 Note the extra C<--> to separate options for the script wrapper from options for the
263 underlying L<Net::Server> options.
265 Note: this script has to be run in the same directory as B<oils_header.pl>.
267 Typical execution will include a trailing C<&> to run in the background.
271 This script is a L<Net::Server::PreFork> instance for shoving records into Evergreen from a remote system.
275 The only required option is --password
277 --password =<eg_password>
278 --user =<eg_username> default: admin
279 --source =<bib_source> default: 1 Integer
280 --merge =<i> default: 0
281 --tempdir =</temp/dir/> default: from L<opensrf.conf> <open-ils.vandelay/app_settings/databases/importer>
282 --source =<i> default: 1
284 =head2 Old style: --noqueue and associated options
286 To bypass vandelay queue processing and push directly into the database (as the old style)
288 --noqueue default: OFF
289 --buffsize =<i> default: 4096 Buffer size. Only used by --noqueue
290 --wait =<i> default: 5 Seconds to read socket before processing. Only used by --noqueue
292 =head2 Net::Server Options
294 By default, the script will use the Net::Server configuration file B<marc_stream_importer.conf>. You can
295 override this by passing a filepath with the --conf_file option.
297 Other Net::Server options include: --port=<port> --min_servers=<X> --max_servers=<Y> and --log_file=[path/to/file]
299 See L<Net::Server> for a complete list.
303 =head3 OCLC Connexion
305 To use this script with OCLC Connexion, configure the client as follows:
307 Under Tools -> Options -> Export (tab)
308 Create -> Choose Connection -> OK -> Leave translation at "None"
309 -> Create -> Create -> choose TCP/IP (internet)
310 -> Enter hostname and Port, leave 'Use Telnet Protocol' checked
311 -> Create/OK your way out of the dialogs
312 Record Characteristics (button) -> Choose 'UTF-8 Unicode' for the Character Set
315 OCLC and Connexion are trademark/service marks of OCLC Online Computer Library Center, Inc.
319 WARNING: This script provides no inherent security layer. Any client that has
320 access to the server+port can inject MARC records into the system.
321 Use the available options (like allow/deny) in the Net::Server config file
322 or via the command line to restrict access as necessary.
326 ./marc_stream_importer.pl \
327 admin open-ils connexion --port 5555 --min_servers 2 \
328 --max_servers=20 --log_file=/openils/var/log/marc_net_importer.log &
332 L<Net::Server::PreFork>, L<marc_stream_importer.conf>
336 Bill Erickson <erickson@esilibrary.com>
337 Joe Atzberger <jatzberger@esilibrary.com>