1 package OpenILS::Application::Vandelay;
2 use strict; use warnings;
3 use OpenILS::Application;
4 use base qw/OpenILS::Application/;
6 use Unicode::Normalize;
7 use OpenSRF::EX qw/:try/;
9 use OpenSRF::AppSession;
10 use OpenSRF::Utils::SettingsClient;
11 use OpenSRF::Utils::Cache;
13 use OpenILS::Utils::Fieldmapper;
14 use OpenILS::Utils::CStoreEditor qw/:funcs/;
20 use OpenILS::Utils::Fieldmapper;
22 use Time::HiRes qw(time);
24 use OpenSRF::Utils::Logger qw/$logger/;
26 use OpenILS::Application::AppUtils;
27 my $U = 'OpenILS::Application::AppUtils';
34 my $form = shift || '';
42 $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
46 # --------------------------------------------------------------------------------
49 sub create_bib_queue {
57 my $e = new_editor(authtoken => $auth, xact => 1);
59 return $e->die_event unless $e->checkauth;
60 return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE');
61 $owner ||= $e->requestor->id;
63 my $queue = new Fieldmapper::vandelay::bib_queue();
64 $queue->name( $name );
65 $queue->owner( $owner );
66 $queue->queue_type( $type ) if ($type);
68 my $new_q = $e->create_vandelay_bib_queue( $queue );
69 return $e->die_event unless ($new_q);
74 __PACKAGE__->register_method(
75 api_name => "open-ils.vandelay.bib_queue.create",
76 method => "create_bib_queue",
82 sub create_auth_queue {
90 my $e = new_editor(authtoken => $auth, xact => 1);
92 return $e->die_event unless $e->checkauth;
93 return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE');
94 $owner ||= $e->requestor->id;
96 my $queue = new Fieldmapper::vandelay::authority_queue();
97 $queue->name( $name );
98 $queue->owner( $owner );
99 $queue->queue_type( $type ) if ($type);
101 my $new_q = $e->create_vandelay_authority_queue( $queue );
102 $e->die_event unless ($new_q);
107 __PACKAGE__->register_method(
108 api_name => "open-ils.vandelay.authority_queue.create",
109 method => "create_auth_queue",
114 sub add_record_to_bib_queue {
122 my $e = new_editor(authtoken => $auth, xact => 1);
124 $queue = $e->retrieve_vandelay_bib_queue($queue);
126 return $e->die_event unless $e->checkauth;
127 return $e->die_event unless
128 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
129 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
131 my $new_rec = _add_bib_rec($e, $marc, $queue->id, $purpose);
133 return $e->die_event unless ($new_rec);
137 __PACKAGE__->register_method(
138 api_name => "open-ils.vandelay.queued_bib_record.create",
139 method => "add_record_to_bib_queue",
150 my $rec = new Fieldmapper::vandelay::queued_bib_record();
152 $rec->queue( $queue );
153 $rec->purpose( $purpose ) if ($purpose);
155 return $e->create_vandelay_queued_bib_record( $rec );
158 sub add_record_to_authority_queue {
166 my $e = new_editor(authtoken => $auth, xact => 1);
168 $queue = $e->retrieve_vandelay_authority_queue($queue);
170 return $e->die_event unless $e->checkauth;
171 return $e->die_event unless
172 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
173 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
175 my $new_rec = _add_auth_rec($e, $marc, $queue->id, $purpose);
177 return $e->die_event unless ($new_rec);
181 __PACKAGE__->register_method(
182 api_name => "open-ils.vandelay.queued_authority_record.create",
183 method => "add_record_to_authority_queue",
194 my $rec = new Fieldmapper::vandelay::queued_authority_record();
196 $rec->queue( $queue );
197 $rec->purpose( $purpose ) if ($purpose);
199 return $e->create_vandelay_queued_authority_record( $rec );
206 my $fingerprint = shift;
207 my $queue_id = shift;
209 my $e = new_editor(authtoken => $auth, xact => 1);
210 return $e->die_event unless $e->checkauth;
213 my $type = $self->{record_type};
216 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
218 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
221 my $evt = check_queue_perms($e, $type, $queue);
224 my $method = "open-ils.vandelay.queued_${type}_record.create";
225 $method = $self->method_lookup( $method );
227 my $cache = new OpenSRF::Utils::Cache();
229 my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint);
230 my $purpose = $data->{purpose};
231 $data = decode_base64($data->{marc});
233 $logger->info("vandelay loaded $fingerprint purpose=$purpose and ".length($data)." bytes of data");
236 open $fh, '<', \$data;
238 my $marctype = 'USMARC'; # ?
239 my $batch = new MARC::Batch ( $marctype, $fh );
243 while (my $r = $batch->next) {
244 $logger->info("processing record $count");
246 (my $xml = $r->as_xml_record()) =~ s/\n//sog;
247 $xml =~ s/^<\?xml.+\?\s*>//go;
248 $xml =~ s/>\s+</></go;
249 $xml =~ s/\p{Cc}//go;
250 $xml = entityize($xml);
251 $xml =~ s/[\x00-\x1f]//go;
253 if ($type eq 'bib') {
254 _add_bib_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
256 _add_auth_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
260 $client->respond( $count );
263 $logger->warn("Encountered a bad record at Vandelay ingest: ".$error);
270 __PACKAGE__->register_method(
271 api_name => "open-ils.vandelay.bib.process_spool",
272 method => "process_spool",
277 __PACKAGE__->register_method(
278 api_name => "open-ils.vandelay.auth.process_spool",
279 method => "process_spool",
282 record_type => 'auth'
286 __PACKAGE__->register_method(
287 api_name => "open-ils.vandelay.bib_queue.records.retrieve",
288 method => 'retrieve_queue',
294 __PACKAGE__->register_method(
295 api_name => "open-ils.vandelay.auth_queue.records.retrieve",
296 method => 'retrieve_queue',
300 record_type => 'auth'
304 my($self, $conn, $auth, $queue_id, $options) = @_;
305 my $e = new_editor(authtoken => $auth);
306 return $e->event unless $e->checkauth;
308 my $type = $self->{record_type};
311 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
313 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
315 my $evt = check_queue_perms($e, $type, $queue);
318 my $class = ($type eq 'bib') ? 'vqbr' : 'vqar';
319 my $search = ($type eq 'bib') ? 'search_vandelay_queued_bib_record' : 'search_vandelay_queued_authority_record';
320 my $retrieve = ($type eq 'bib') ? 'retrieve_vandelay_queued_bib_record' : 'retrieve_vandelay_queued_authority_record';
321 my $record_ids = $e->$search({queue => $queue_id}, {idlist => 1});
323 for my $rec_id (@$record_ids) {
324 my $rec = $e->$retrieve([
327 flesh_fields => {$class => ['attributes', 'matches']}
330 $rec->clear_marc if $$options{clear_marc};
331 $conn->respond($rec);
336 sub check_queue_perms {
337 my($e, $type, $queue) = @_;
338 if ($type eq 'bib') {
339 return $e->die_event unless
340 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
341 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
343 return $e->die_event unless
344 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
345 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
351 __PACKAGE__->register_method(
352 api_name => "open-ils.vandelay.bib_record.list.import",
353 method => 'import_record_list',
360 __PACKAGE__->register_method(
361 api_name => "open-ils.vandelay.auth_record.list.import",
362 method => 'import_record_list',
366 record_type => 'auth'
369 sub import_record_list {
370 my($self, $conn, $auth, $rec_ids, $args) = @_;
371 my $e = new_editor(xact => 1, authtoken => $auth);
372 return $e->die_event unless $e->checkauth;
374 my $err = import_record_list_impl($self, $conn, $auth, $e, $rec_ids, $args);
377 return {complete => 1};
380 sub import_record_list_impl {
381 my($self, $conn, $auth, $e, $rec_ids, $args) = @_;
383 my $overlay_map = $args->{overlay_map} || {};
384 my $type = $self->{record_type};
385 my $total = @$rec_ids;
389 for my $rec_id (@$rec_ids) {
392 my $rec = $e->retrieve_vandelay_queued_bib_record($rec_id)
393 or return $e->die_event;
395 $queues{$rec->queue} = 1;
398 if(defined $overlay_map->{$rec_id}) {
399 $logger->info("vl: overlaying record $rec_id");
400 $record = $U->simplereq(
402 'open-ils.cat.biblio.record.xml.update',
403 $auth, $overlay_map->{$rec_id}, $rec->marc); #$rec->bib_source);
405 $logger->info("vl: importing new record");
406 $record = $U->simplereq(
408 'open-ils.cat.biblio.record.xml.import',
409 $auth, $rec->marc); #$rec->bib_source);
412 if($U->event_code($record)) {
417 $rec->imported_as($record->id);
418 $rec->import_time('now');
419 $e->update_vandelay_queued_bib_record($rec) or return $e->die_event;
422 $conn->respond({total => $total, progress => ++$count, imported => $rec_id});
425 # see if we need to mark any queues as complete
426 for my $q_id (keys %queues) {
428 my $remaining = $e->search_vandelay_queued_bib_record(
429 {queue => $q_id, import_time => undef}, {idlist => 1});
430 unless(@$remaining) {
431 my $queue = $e->retrieve_vandelay_bib_queue($q_id);
432 unless($U->is_true($queue->complete)) {
433 $queue->complete('t');
434 $e->update_vandelay_bib_queue($queue) or return $e->die_event;
444 __PACKAGE__->register_method(
445 api_name => "open-ils.vandelay.bib_queue.owner.retrieve",
446 method => 'owner_queue_retrieve',
452 __PACKAGE__->register_method(
453 api_name => "open-ils.vandelay.authority_queue.owner.retrieve",
454 method => 'owner_queue_retrieve',
458 record_type => 'auth'
461 sub owner_queue_retrieve {
462 my($self, $conn, $auth, $owner_id) = @_;
463 my $e = new_editor(authtoken => $auth);
464 return $e->die_event unless $e->checkauth;
465 $owner_id = $e->requestor->id; # XXX add support for viewing other's queues
467 if($self->{record_type} eq 'bib') {
468 $queues = $e->search_vandelay_bib_queue(
469 {complete => 'f', owner => $owner_id});
471 $queues = $e->search_vandelay_authority_queue(
472 {complete => 'f', owner => $owner_id});
474 $conn->respond($_) for @$queues;