1 package OpenILS::Application::Vandelay;
2 use strict; use warnings;
3 use OpenILS::Application;
4 use base qw/OpenILS::Application/;
5 use Unicode::Normalize;
6 use OpenSRF::EX qw/:try/;
7 use OpenSRF::AppSession;
8 use OpenSRF::Utils::SettingsClient;
9 use OpenSRF::Utils::Cache;
10 use OpenILS::Utils::Fieldmapper;
11 use OpenILS::Utils::CStoreEditor qw/:funcs/;
15 use OpenILS::Utils::Fieldmapper;
16 use Time::HiRes qw(time);
17 use OpenSRF::Utils::Logger qw/$logger/;
19 use OpenILS::Application::AppUtils;
20 use OpenILS::Application::Cat::BibCommon;
21 use OpenILS::Application::Cat::AuthCommon;
22 my $U = 'OpenILS::Application::AppUtils';
27 # --------------------------------------------------------------------------------
30 sub create_bib_queue {
38 my $e = new_editor(authtoken => $auth, xact => 1);
40 return $e->die_event unless $e->checkauth;
41 return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE');
42 $owner ||= $e->requestor->id;
44 return OpenILS::Event->new('BIB_QUEUE_EXISTS')
45 if $e->search_vandelay_bib_queue(
46 {name => $name, owner => $owner, queue_type => $type})->[0];
48 my $queue = new Fieldmapper::vandelay::bib_queue();
49 $queue->name( $name );
50 $queue->owner( $owner );
51 $queue->queue_type( $type ) if ($type);
53 my $new_q = $e->create_vandelay_bib_queue( $queue );
54 return $e->die_event unless ($new_q);
59 __PACKAGE__->register_method(
60 api_name => "open-ils.vandelay.bib_queue.create",
61 method => "create_bib_queue",
67 sub create_auth_queue {
75 my $e = new_editor(authtoken => $auth, xact => 1);
77 return $e->die_event unless $e->checkauth;
78 return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE');
79 $owner ||= $e->requestor->id;
81 return OpenILS::Event->new('AUTH_QUEUE_EXISTS')
82 if $e->search_vandelay_bib_queue(
83 {name => $name, owner => $owner, queue_type => $type})->[0];
85 my $queue = new Fieldmapper::vandelay::authority_queue();
86 $queue->name( $name );
87 $queue->owner( $owner );
88 $queue->queue_type( $type ) if ($type);
90 my $new_q = $e->create_vandelay_authority_queue( $queue );
91 $e->die_event unless ($new_q);
96 __PACKAGE__->register_method(
97 api_name => "open-ils.vandelay.authority_queue.create",
98 method => "create_auth_queue",
103 sub add_record_to_bib_queue {
111 my $e = new_editor(authtoken => $auth, xact => 1);
113 $queue = $e->retrieve_vandelay_bib_queue($queue);
115 return $e->die_event unless $e->checkauth;
116 return $e->die_event unless
117 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
118 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
120 my $new_rec = _add_bib_rec($e, $marc, $queue->id, $purpose);
122 return $e->die_event unless ($new_rec);
126 __PACKAGE__->register_method(
127 api_name => "open-ils.vandelay.queued_bib_record.create",
128 method => "add_record_to_bib_queue",
139 my $rec = new Fieldmapper::vandelay::queued_bib_record();
141 $rec->queue( $queue );
142 $rec->purpose( $purpose ) if ($purpose);
144 return $e->create_vandelay_queued_bib_record( $rec );
147 sub add_record_to_authority_queue {
155 my $e = new_editor(authtoken => $auth, xact => 1);
157 $queue = $e->retrieve_vandelay_authority_queue($queue);
159 return $e->die_event unless $e->checkauth;
160 return $e->die_event unless
161 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
162 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
164 my $new_rec = _add_auth_rec($e, $marc, $queue->id, $purpose);
166 return $e->die_event unless ($new_rec);
170 __PACKAGE__->register_method(
171 api_name => "open-ils.vandelay.queued_authority_record.create",
172 method => "add_record_to_authority_queue",
183 my $rec = new Fieldmapper::vandelay::queued_authority_record();
185 $rec->queue( $queue );
186 $rec->purpose( $purpose ) if ($purpose);
188 return $e->create_vandelay_queued_authority_record( $rec );
195 my $fingerprint = shift;
196 my $queue_id = shift;
198 my $e = new_editor(authtoken => $auth, xact => 1);
199 return $e->die_event unless $e->checkauth;
202 my $type = $self->{record_type};
205 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
207 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
210 my $evt = check_queue_perms($e, $type, $queue);
213 my $method = "open-ils.vandelay.queued_${type}_record.create";
214 $method = $self->method_lookup( $method );
216 my $cache = new OpenSRF::Utils::Cache();
218 my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint);
219 my $purpose = $data->{purpose};
220 my $filename = $data->{path};
222 unless(-r $filename) {
223 $logger->error("unable to read MARC file");
224 return -1; # make this an event XXX
227 $logger->info("vandelay spooling $fingerprint purpose=$purpose file=$filename");
229 my $marctype = 'USMARC'; # ?
230 my $batch = new MARC::Batch ($marctype, $filename);
235 while (try { $r = $batch->next } otherwise { $r = -1 }) {
237 $logger->warn("Proccessing of record $count in set $fingerprint failed. Skipping this record");
241 $logger->info("processing record $count");
244 (my $xml = $r->as_xml_record()) =~ s/\n//sog;
245 $xml =~ s/^<\?xml.+\?\s*>//go;
246 $xml =~ s/>\s+</></go;
247 $xml =~ s/\p{Cc}//go;
248 $xml = $U->entityize($xml);
249 $xml =~ s/[\x00-\x1f]//go;
251 if ($type eq 'bib') {
252 _add_bib_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
254 _add_auth_rec( $e, $xml, $queue_id, $purpose ) or return $e->die_event;
258 $client->respond( $count );
261 $logger->warn("Encountered a bad record at Vandelay ingest: ".$error);
267 $cache->delete_cache('vandelay_import_spool_' . $fingerprint);
271 __PACKAGE__->register_method(
272 api_name => "open-ils.vandelay.bib.process_spool",
273 method => "process_spool",
278 __PACKAGE__->register_method(
279 api_name => "open-ils.vandelay.auth.process_spool",
280 method => "process_spool",
283 record_type => 'auth'
287 __PACKAGE__->register_method(
288 api_name => "open-ils.vandelay.bib_queue.records.retrieve",
289 method => 'retrieve_queued_records',
295 __PACKAGE__->register_method(
296 api_name => "open-ils.vandelay.auth_queue.records.retrieve",
297 method => 'retrieve_queued_records',
301 record_type => 'auth'
304 __PACKAGE__->register_method(
305 api_name => "open-ils.vandelay.bib_queue.records.matches.retrieve",
306 method => 'retrieve_queued_records',
310 record_type => 'bib',
312 desc => q/Only retrieve queued bib records that have matches against existing records/
315 __PACKAGE__->register_method(
316 api_name => "open-ils.vandelay.auth_queue.records.matches.retrieve",
317 method => 'retrieve_queued_records',
321 record_type => 'auth',
323 desc => q/Only retrieve queued authority records that have matches against existing records/
328 sub retrieve_queued_records {
329 my($self, $conn, $auth, $queue_id, $options) = @_;
330 my $e = new_editor(authtoken => $auth);
331 return $e->event unless $e->checkauth;
333 my $limit = $$options{limit} || 20;
334 my $offset = $$options{offset} || 0;
336 my $type = $self->{record_type};
339 $queue = $e->retrieve_vandelay_bib_queue($queue_id) or return $e->die_event;
341 $queue = $e->retrieve_vandelay_authority_queue($queue_id) or return $e->die_event;
343 my $evt = check_queue_perms($e, $type, $queue);
346 my $class = ($type eq 'bib') ? 'vqbr' : 'vqar';
347 my $search = ($type eq 'bib') ?
348 'search_vandelay_queued_bib_record' : 'search_vandelay_queued_authority_record';
349 my $retrieve = ($type eq 'bib') ?
350 'retrieve_vandelay_queued_bib_record' : 'retrieve_vandelay_queued_authority_record';
352 my $filter = ($$options{non_imported}) ? {import_time => undef} : {};
355 if($self->api_name =~ /matches/) {
356 # fetch only matched records
357 $record_ids = queued_records_with_matches($e, $type, $queue_id, $limit, $offset, $filter);
359 # fetch all queue records
360 $record_ids = $e->$search([
361 {queue => $queue_id, %$filter},
362 {order_by => {$class => 'id'}, limit => $limit, offset => $offset}
369 for my $rec_id (@$record_ids) {
372 flesh_fields => {$class => ['attributes', 'matches']},
374 my $rec = $e->$retrieve([$rec_id, $params]);
375 $rec->clear_marc if $$options{clear_marc};
376 $conn->respond($rec);
381 sub check_queue_perms {
382 my($e, $type, $queue) = @_;
383 if ($type eq 'bib') {
384 return $e->die_event unless
385 ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) ||
386 $e->allowed('CREATE_BIB_IMPORT_QUEUE'));
388 return $e->die_event unless
389 ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) ||
390 $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE'));
396 __PACKAGE__->register_method(
397 api_name => "open-ils.vandelay.bib_record.list.import",
398 method => 'import_record_list',
405 __PACKAGE__->register_method(
406 api_name => "open-ils.vandelay.auth_record.list.import",
407 method => 'import_record_list',
411 record_type => 'auth'
414 sub import_record_list {
415 my($self, $conn, $auth, $rec_ids, $args) = @_;
416 my $e = new_editor(authtoken => $auth);
417 return $e->event unless $e->checkauth;
419 my $err = import_record_list_impl($self, $conn, $rec_ids, $e->requestor, $args);
421 return {complete => 1};
425 __PACKAGE__->register_method(
426 api_name => "open-ils.vandelay.bib_queue.import",
427 method => 'import_queue',
434 __PACKAGE__->register_method(
435 api_name => "open-ils.vandelay.auth_queue.import",
436 method => 'import_queue',
440 record_type => 'auth'
442 __PACKAGE__->register_method(
443 api_name => "open-ils.vandelay.bib_queue.nomatch.import",
444 method => 'import_queue',
449 desc => q/Only import records that have no collisions/
454 __PACKAGE__->register_method(
455 api_name => "open-ils.vandelay.auth_queue.nomatch.import",
456 method => 'import_queue',
461 desc => q/Only import records that have no collisions/
463 record_type => 'auth'
466 my($self, $conn, $auth, $q_id, $options) = @_;
467 my $e = new_editor(authtoken => $auth);
468 return $e->event unless $e->checkauth;
470 my $type = $self->{record_type};
471 my $class = ($type eq 'bib') ? 'vqbr' : 'vqar';
473 my $query = {queue => $q_id, import_time => undef};
475 if($self->api_name =~ /nomatch/) {
476 my $matched_recs = queued_records_with_matches($e, $type, $q_id, undef, undef, {import_time => undef});
477 $query->{id} = {'not in' => $matched_recs} if @$matched_recs;
480 my $search = ($type eq 'bib') ?
481 'search_vandelay_queued_bib_record' : 'search_vandelay_queued_authority_record';
482 my $rec_ids = $e->$search($query, {idlist => 1});
483 my $err = import_record_list_impl($self, $conn, $rec_ids, $e->requestor, $options);
485 return {complete => 1};
489 # returns a list of queued record IDs for a given queue that
490 # have at least one entry in the match table
491 sub queued_records_with_matches {
492 my($e, $type, $q_id, $limit, $offset, $filter) = @_;
494 my $match_class = 'vbm';
495 my $rec_class = 'vqbr';
496 if($type eq 'auth') {
497 $match_class = 'vam';
502 $filter->{queue} = $q_id;
506 select => {$match_class => ['queued_record']},
511 fkey => 'queued_record',
518 if($limit or defined $offset) {
521 $query->{limit} = $limit;
522 $query->{offset} = $offset;
525 my $data = $e->json_query($query);
526 return [ map {$_->{queued_record}} @$data ];
529 sub import_record_list_impl {
530 my($self, $conn, $rec_ids, $requestor, $args) = @_;
532 my $overlay_map = $args->{overlay_map} || {};
533 my $type = $self->{record_type};
534 my $total = @$rec_ids;
538 for my $rec_id (@$rec_ids) {
540 my $overlay_target = $overlay_map->{$rec_id};
542 my $e = new_editor(xact => 1);
543 $e->requestor($requestor);
547 my $rec = $e->retrieve_vandelay_queued_bib_record($rec_id) ;
549 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $e->die_event});
553 next if $rec->import_time;
554 $queues{$rec->queue} = 1;
557 if(defined $overlay_target) {
558 $logger->info("vl: overlaying record $overlay_target");
559 $record = OpenILS::Application::Cat::BibCommon->biblio_record_replace_marc(
560 $e, $overlay_target, $rec->marc); #$rec->bib_source
562 $logger->info("vl: importing new record");
563 $record = OpenILS::Application::Cat::BibCommon->biblio_record_xml_import(
564 $e, $rec->marc); #$rec->bib_source
567 if($U->event_code($record)) {
568 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $record});
571 $rec->imported_as($record->id);
572 $rec->import_time('now');
574 unless($e->update_vandelay_queued_bib_record($rec)) {
575 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $e->die_event});
581 my $rec = $e->retrieve_vandelay_queued_authority_record($rec_id);
583 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $e->die_event});
587 next if $rec->import_time;
588 $queues{$rec->queue} = 1;
591 if(defined $overlay_target) {
592 $logger->info("vl: overlaying record $overlay_target");
593 $record = OpenILS::Utils::Cat::AuthCommon->overlay_authority_record(
594 $overlay_target, $rec->marc); #$source);
596 $logger->info("vl: importing new record");
597 $record = OpenILS::Utils::Cat::AuthCommon->import_authority_record(
598 $rec->marc) #$source);
601 if($U->event_code($record)) {
602 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $record});
606 $rec->imported_as($record->id);
607 $rec->import_time('now');
608 unless($e->update_vandelay_queued_authority_record($rec)) {
609 $conn->respond({total => $total, progress => ++$count, imported => $rec_id, err_event => $e->die_event});
615 $conn->respond({total => $total, progress => ++$count, imported => $rec_id});
618 # see if we need to mark any queues as complete
619 my $e = new_editor(xact => 1);
620 for my $q_id (keys %queues) {
622 my $remaining = $e->search_vandelay_queued_bib_record(
623 [{queue => $q_id, import_time => undef}, {limit =>1}], {idlist => 1});
624 unless(@$remaining) {
625 my $queue = $e->retrieve_vandelay_bib_queue($q_id);
626 unless($U->is_true($queue->complete)) {
627 $queue->complete('t');
628 $e->update_vandelay_bib_queue($queue) or return $e->die_event;
634 my $remaining = $e->search_vandelay_queued_authority_record(
635 [{queue => $q_id, import_time => undef}, {limit =>1}], {idlist => 1});
636 unless(@$remaining) {
637 my $queue = $e->retrieve_vandelay_authority_queue($q_id);
638 unless($U->is_true($queue->complete)) {
639 $queue->complete('t');
640 $e->update_vandelay_authority_queue($queue) or return $e->die_event;
653 __PACKAGE__->register_method(
654 api_name => "open-ils.vandelay.bib_queue.owner.retrieve",
655 method => 'owner_queue_retrieve',
661 __PACKAGE__->register_method(
662 api_name => "open-ils.vandelay.authority_queue.owner.retrieve",
663 method => 'owner_queue_retrieve',
667 record_type => 'auth'
670 sub owner_queue_retrieve {
671 my($self, $conn, $auth, $owner_id, $filters) = @_;
672 my $e = new_editor(authtoken => $auth);
673 return $e->die_event unless $e->checkauth;
674 $owner_id = $e->requestor->id; # XXX add support for viewing other's queues?
677 my $search = {owner => $owner_id};
678 $search->{$_} = $filters->{$_} for keys %$filters;
680 if($self->{record_type} eq 'bib') {
681 $queues = $e->search_vandelay_bib_queue(
682 [$search, {order_by => {vbq => 'lower(name)'}}]);
684 $queues = $e->search_vandelay_authority_queue(
685 [$search, {order_by => {vaq => 'lower(name)'}}]);
687 $conn->respond($_) for @$queues;
691 __PACKAGE__->register_method(
692 api_name => "open-ils.vandelay.bib_queue.delete",
693 method => "delete_queue",
698 __PACKAGE__->register_method(
699 api_name => "open-ils.vandelay.auth_queue.delete",
700 method => "delete_queue",
703 record_type => 'auth'
707 my($self, $conn, $auth, $q_id) = @_;
708 my $e = new_editor(xact => 1, authtoken => $auth);
709 return $e->die_event unless $e->checkauth;
710 if($self->{record_type} eq 'bib') {
711 return $e->die_event unless $e->allowed('CREATE_BIB_IMPORT_QUEUE');
712 my $queue = $e->retrieve_vandelay_bib_queue($q_id)
713 or return $e->die_event;
714 $e->delete_vandelay_bib_queue($queue)
715 or return $e->die_event;
717 return $e->die_event unless $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE');
718 my $queue = $e->retrieve_vandelay_authority_queue($q_id)
719 or return $e->die_event;
720 $e->delete_vandelay_authority_queue($queue)
721 or return $e->die_event;
728 __PACKAGE__->register_method(
729 api_name => "open-ils.vandelay.queued_bib_record.html",
730 method => 'queued_record_html',
736 __PACKAGE__->register_method(
737 api_name => "open-ils.vandelay.queued_authority_record.html",
738 method => 'queued_record_html',
742 record_type => 'auth'
745 sub queued_record_html {
746 my($self, $conn, $auth, $rec_id) = @_;
747 my $e = new_editor(authtoken => $auth);
748 return $e->event unless $e->checkauth;
750 if($self->{record_type} eq 'bib') {
751 $rec = $e->retrieve_vandelay_queued_bib_record($rec_id)
754 $rec = $e->retrieve_vandelay_queued_authority_record($rec_id)
758 return $U->simplereq(
760 'open-ils.search.biblio.record.html', undef, 1, $rec->marc);
764 __PACKAGE__->register_method(
765 api_name => "open-ils.vandelay.bib_queue.summary.retrieve",
766 method => 'retrieve_queue_summary',
772 __PACKAGE__->register_method(
773 api_name => "open-ils.vandelay.auth_queue.summary.retrieve",
774 method => 'retrieve_queue_summary',
778 record_type => 'auth'
781 sub retrieve_queue_summary {
782 my($self, $conn, $auth, $queue_id) = @_;
783 my $e = new_editor(authtoken => $auth);
784 return $e->event unless $e->checkauth;
787 my $type = $self->{record_type};
789 $queue = $e->retrieve_vandelay_bib_queue($queue_id)
792 $queue = $e->retrieve_vandelay_authority_queue($queue_id)
796 my $evt = check_queue_perms($e, $type, $queue);
799 my $search = 'search_vandelay_queued_bib_record';
800 $search =~ s/bib/authority/ if $type ne 'bib';
804 total => scalar(@{$e->$search({queue => $queue_id}, {idlist=>1})}),
805 imported => scalar(@{$e->$search({queue => $queue_id, import_time => {'!=' => undef}}, {idlist=>1})}),