1 package OpenILS::Application::Vandelay;
2 use strict; use warnings;
3 use OpenILS::Application;
4 use base qw/OpenILS::Application/;
6 use Unicode::Normalize;
7 use OpenSRF::EX qw/:try/;
9 use OpenSRF::AppSession;
10 use OpenSRF::Utils::SettingsClient;
11 use OpenSRF::Utils::Cache;
13 use OpenILS::Utils::Fieldmapper;
14 use OpenILS::Utils::CStoreEditor qw/:funcs/;
20 use OpenILS::Utils::Fieldmapper;
22 use Time::HiRes qw(time);
24 use OpenSRF::Utils::Logger qw/$logger/;
26 use OpenILS::Application::AppUtils;
27 my $U = 'OpenILS::Application::AppUtils';
32 # --------------------------------------------------------------------------------
35 sub create_bib_queue {
43 my $e = new_editor(authtoken => $auth, xact => 1);
45 return $e->die_event unless $e->checkauth;
46 return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE');
47 $owner ||= $e->requestor->id;
49 my $queue = new Fieldmapper::vandelay::bib_queue();
50 $queue->name( $name );
51 $queue->owner( $owner );
52 $queue->queue_type( $type ) if ($type);
54 my $new_q = $e->create_vandelay_bib_queue( $queue );
55 return $e->die_event unless ($new_q);
60 __PACKAGE__->register_method(
61 api_name => "open-ils.vandelay.bib_queue.create",
62 method => "create_bib_queue",
68 sub create_auth_queue {
76 my $e = new_editor(authtoken => $auth, xact => 1);
78 return $e->die_event unless $e->checkauth;
79 return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE');
80 $owner ||= $e->requestor->id;
82 my $queue = new Fieldmapper::vandelay::authority_queue();
83 $queue->name( $name );
84 $queue->owner( $owner );
85 $queue->queue_type( $type ) if ($type);
87 my $new_q = $e->create_vandelay_authority_queue( $queue );
88 $e->die_event unless ($new_q);
93 __PACKAGE__->register_method(
94 api_name => "open-ils.vandelay.authority_queue.create",
95 method => "create_auth_queue",
100 sub add_record_to_bib_queue {
108 my $e = new_editor(authtoken => $auth, xact => 1);
110 $queue = $e->retrieve_vandelay_bib_queue($queue);
112 return $e->die_event unless $e->checkauth;
113 return $e->die_event unless
114 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
115 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
117 my $new_rec = _add_bib_rec($e, $marc, $queue->id, $purpose);
119 return $e->die_event unless ($new_rec);
123 __PACKAGE__->register_method(
124 api_name => "open-ils.vandelay.queued_bib_record.create",
125 method => "add_record_to_bib_queue",
136 my $rec = new Fieldmapper::vandelay::queued_bib_record();
138 $rec->queue( $queue );
139 $rec->purpose( $purpose ) if ($purpose);
141 return $e->create_vandelay_queued_bib_record( $rec );
144 sub add_record_to_authority_queue {
152 my $e = new_editor(authtoken => $auth, xact => 1);
154 $queue = $e->retrieve_vandelay_authority_queue($queue);
156 return $e->die_event unless $e->checkauth;
157 return $e->die_event unless
158 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
159 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
161 my $new_rec = _add_auth_rec($e, $marc, $queue->id, $purpose);
163 return $e->die_event unless ($new_rec);
167 __PACKAGE__->register_method(
168 api_name => "open-ils.vandelay.queued_authority_record.create",
169 method => "add_record_to_authority_queue",
180 my $rec = new Fieldmapper::vandelay::queued_authority_record();
182 $rec->queue( $queue );
183 $rec->purpose( $purpose ) if ($purpose);
185 return $e->create_vandelay_queued_authority_record( $rec );
192 my $fingerprint = shift;
193 my $queue_id = shift;
195 my $e = new_editor(authtoken => $auth, xact => 1);
196 return $e->die_event unless $e->checkauth;
199 my $type = $self->{record_type};
202 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
204 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
207 my $evt = check_queue_perms($e, $type, $queue);
210 my $method = "open-ils.vandelay.queued_${type}_record.create";
211 $method = $self->method_lookup( $method );
213 my $cache = new OpenSRF::Utils::Cache();
215 my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint);
216 my $purpose = $data->{purpose};
217 $data = decode_base64($data->{marc});
219 $logger->info("vandelay loaded $fingerprint purpose=$purpose and ".length($data)." bytes of data");
222 open $fh, '<', \$data;
224 my $marctype = 'USMARC'; # ?
225 my $batch = new MARC::Batch ( $marctype, $fh );
229 while (my $r = $batch->next) {
230 $logger->info("processing record $count");
232 (my $xml = $r->as_xml_record()) =~ s/\n//sog;
233 $xml =~ s/^<\?xml.+\?\s*>//go;
234 $xml =~ s/>\s+</></go;
235 $xml =~ s/\p{Cc}//go;
236 $xml = $U->entityize($xml);
237 $xml =~ s/[\x00-\x1f]//go;
239 if ($type eq 'bib') {
240 _add_bib_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
242 _add_auth_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
246 $client->respond( $count );
249 $logger->warn("Encountered a bad record at Vandelay ingest: ".$error);
256 __PACKAGE__->register_method(
257 api_name => "open-ils.vandelay.bib.process_spool",
258 method => "process_spool",
263 __PACKAGE__->register_method(
264 api_name => "open-ils.vandelay.auth.process_spool",
265 method => "process_spool",
268 record_type => 'auth'
272 __PACKAGE__->register_method(
273 api_name => "open-ils.vandelay.bib_queue.records.retrieve",
274 method => 'retrieve_queue',
280 __PACKAGE__->register_method(
281 api_name => "open-ils.vandelay.auth_queue.records.retrieve",
282 method => 'retrieve_queue',
286 record_type => 'auth'
289 __PACKAGE__->register_method(
290 api_name => "open-ils.vandelay.bib_queue.records.matches.retrieve",
291 method => 'retrieve_queue',
295 record_type => 'bib',
297 desc => q/Only retrieve queued bib records that have matches against existing records/
300 __PACKAGE__->register_method(
301 api_name => "open-ils.vandelay.auth_queue.records.matches.retrieve",
302 method => 'retrieve_queue',
306 record_type => 'auth',
308 desc => q/Only retrieve queued authority records that have matches against existing records/
314 my($self, $conn, $auth, $queue_id, $options) = @_;
315 my $e = new_editor(authtoken => $auth);
316 return $e->event unless $e->checkauth;
318 my $limit = $$options{limit} || 20;
319 my $offset = $$options{offset} || 0;
321 my $type = $self->{record_type};
324 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
326 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
328 my $evt = check_queue_perms($e, $type, $queue);
331 my $class = ($type eq 'bib') ? 'vqbr' : 'vqar';
332 my $search = ($type eq 'bib') ?
333 'search_vandelay_queued_bib_record' : 'search_vandelay_queued_authority_record';
334 my $retrieve = ($type eq 'bib') ?
335 'retrieve_vandelay_queued_bib_record' : 'retrieve_vandelay_queued_authority_record';
337 my $record_ids = $e->$search([
338 {queue => $queue_id},
339 {order_by => {$class => 'id'}, limit => $limit, offset => $offset}
344 for my $rec_id (@$record_ids) {
347 flesh_fields => {$class => ['attributes', 'matches']},
349 my $rec = $e->$retrieve([$rec_id, $params]);
350 $rec->clear_marc if $$options{clear_marc};
351 next if $self->api_name =~ /matches/ and not @{$rec->matches};
352 $conn->respond($rec);
357 sub check_queue_perms {
358 my($e, $type, $queue) = @_;
359 if ($type eq 'bib') {
360 return $e->die_event unless
361 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
362 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
364 return $e->die_event unless
365 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
366 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
372 __PACKAGE__->register_method(
373 api_name => "open-ils.vandelay.bib_record.list.import",
374 method => 'import_record_list',
381 __PACKAGE__->register_method(
382 api_name => "open-ils.vandelay.auth_record.list.import",
383 method => 'import_record_list',
387 record_type => 'auth'
390 sub import_record_list {
391 my($self, $conn, $auth, $rec_ids, $args) = @_;
392 my $e = new_editor(xact => 1, authtoken => $auth);
393 return $e->die_event unless $e->checkauth;
395 my $err = import_record_list_impl($self, $conn, $auth, $e, $rec_ids, $args);
398 return {complete => 1};
402 __PACKAGE__->register_method(
403 api_name => "open-ils.vandelay.bib_queue.import",
404 method => 'import_queue',
411 __PACKAGE__->register_method(
412 api_name => "open-ils.vandelay.auth_queue.import",
413 method => 'import_queue',
417 record_type => 'auth'
420 sub import_record_list {
421 my($self, $conn, $auth, $rec_ids, $args) = @_;
422 my $e = new_editor(xact => 1, authtoken => $auth);
423 return $e->die_event unless $e->checkauth;
425 my $err = import_record_list_impl($self, $conn, $auth, $e, $rec_ids, $args);
428 return {complete => 1};
432 sub import_record_list_impl {
433 my($self, $conn, $auth, $e, $rec_ids, $args) = @_;
435 my $overlay_map = $args->{overlay_map} || {};
436 my $type = $self->{record_type};
437 my $total = @$rec_ids;
441 for my $rec_id (@$rec_ids) {
444 my $rec = $e->retrieve_vandelay_queued_bib_record($rec_id)
445 or return $e->die_event;
447 $queues{$rec->queue} = 1;
450 if(defined $overlay_map->{$rec_id}) {
451 $logger->info("vl: overlaying record $rec_id");
452 $record = $U->simplereq(
454 'open-ils.cat.biblio.record.xml.update',
455 $auth, $overlay_map->{$rec_id}, $rec->marc); #$rec->bib_source);
457 $logger->info("vl: importing new record");
458 $record = $U->simplereq(
460 'open-ils.cat.biblio.record.xml.import',
461 $auth, $rec->marc); #$rec->bib_source);
464 if($U->event_code($record)) {
469 $rec->imported_as($record->id);
470 $rec->import_time('now');
471 $e->update_vandelay_queued_bib_record($rec) or return $e->die_event;
475 my $rec = $e->retrieve_vandelay_queued_authority_record($rec_id)
476 or return $e->die_event;
478 $queues{$rec->queue} = 1;
481 if(defined $overlay_map->{$rec_id}) {
482 $logger->info("vl: overlaying record $rec_id");
483 $record = $U->simplereq(
485 'open-ils.cat.authority.record.overlay',
486 $auth, $overlay_map->{$rec_id}, $rec->marc); #$rec->bib_source);
488 $logger->info("vl: importing new record");
489 $record = $U->simplereq(
491 'open-ils.cat.authority.record.import',
492 $auth, $rec->marc); #$rec->bib_source);
495 if($U->event_code($record)) {
500 $rec->imported_as($record->id);
501 $rec->import_time('now');
502 $e->update_vandelay_queued_authority_record($rec) or return $e->die_event;
505 $conn->respond({total => $total, progress => ++$count, imported => $rec_id});
508 # see if we need to mark any queues as complete
509 for my $q_id (keys %queues) {
511 my $remaining = $e->search_vandelay_queued_bib_record(
512 {queue => $q_id, import_time => undef}, {idlist => 1});
513 unless(@$remaining) {
514 my $queue = $e->retrieve_vandelay_bib_queue($q_id);
515 unless($U->is_true($queue->complete)) {
516 $queue->complete('t');
517 $e->update_vandelay_bib_queue($queue) or return $e->die_event;
521 my $remaining = $e->search_vandelay_queued_authority_record(
522 {queue => $q_id, import_time => undef}, {idlist => 1});
523 unless(@$remaining) {
524 my $queue = $e->retrieve_vandelay_authority_queue($q_id);
525 unless($U->is_true($queue->complete)) {
526 $queue->complete('t');
527 $e->update_vandelay_authority_queue($queue) or return $e->die_event;
537 __PACKAGE__->register_method(
538 api_name => "open-ils.vandelay.bib_queue.owner.retrieve",
539 method => 'owner_queue_retrieve',
545 __PACKAGE__->register_method(
546 api_name => "open-ils.vandelay.authority_queue.owner.retrieve",
547 method => 'owner_queue_retrieve',
551 record_type => 'auth'
554 sub owner_queue_retrieve {
555 my($self, $conn, $auth, $owner_id) = @_;
556 my $e = new_editor(authtoken => $auth);
557 return $e->die_event unless $e->checkauth;
558 $owner_id = $e->requestor->id; # XXX add support for viewing other's queues?
560 if($self->{record_type} eq 'bib') {
561 $queues = $e->search_vandelay_bib_queue(
562 {complete => 'f', owner => $owner_id});
564 $queues = $e->search_vandelay_authority_queue(
565 {complete => 'f', owner => $owner_id});
567 $conn->respond($_) for @$queues;
571 __PACKAGE__->register_method(
572 api_name => "open-ils.vandelay.bib_queue.delete",
573 method => "delete_queue",
578 __PACKAGE__->register_method(
579 api_name => "open-ils.vandelay.auth_queue.delete",
580 method => "delete_queue",
583 record_type => 'auth'
587 my($self, $conn, $auth, $q_id) = @_;
588 my $e = new_editor(xact => 1, authtoken => $auth);
589 return $e->die_event unless $e->checkauth;
590 if($self->{record_type} eq 'bib') {
591 return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE');
592 my $queue = $e->retrieve_vandelay_bib_queue($q_id)
593 or return $e->die_event;
594 $e->delete_vandelay_bib_queue($queue)
595 or return $e->die_event;
597 return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE');
598 my $queue = $e->retrieve_vandelay_authority_queue($q_id)
599 or return $e->die_event;
600 $e->delete_vandelay_authority_queue($queue)
601 or return $e->die_event;