Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm

   1 package OpenILS::Application::Ingest;
   2 use OpenILS::Application;
   3 use base qw/OpenILS::Application/;
   4
   5 use Unicode::Normalize;
   6 use OpenSRF::EX qw/:try/;
   7
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenSRF::Utils::Logger qw/:level/;
  11
  12 use OpenILS::Utils::ScriptRunner;
  13 use OpenILS::Utils::Fieldmapper;
  14 use OpenSRF::Utils::JSON;
  15
  16 use OpenILS::Utils::Fieldmapper;
  17
  18 use XML::LibXML;
  19 use XML::LibXSLT;
  20 use Time::HiRes qw(time);
  21
  22 our %supported_formats = (
  23         mods32  => {ns => 'http://www.loc.gov/mods/v3'},
  24         mods3   => {ns => 'http://www.loc.gov/mods/v3'},
  25         mods    => {ns => 'http://www.loc.gov/mods/'},
  26         marcxml => {ns => 'http://www.loc.gov/MARC21/slim'},
  27         srw_dc  => {ns => 'info:srw/schema/1/dc-schema'},
  28         oai_dc  => {ns => 'http://www.openarchives.org/OAI/2.0/oai_dc/'},
  29         rdf_dc  => {ns => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'},
  30         atom    => {ns => 'http://www.w3.org/2005/Atom'},
  31         rss091  => {ns => 'http://my.netscape.com/rdf/simple/0.9/'},
  32         rss092  => {ns => ''},
  33         rss093  => {ns => ''},
  34         rss094  => {ns => ''},
  35         rss10   => {ns => 'http://purl.org/rss/1.0/'},
  36         rss11   => {ns => 'http://purl.org/net/rss1.1#'},
  37         rss2    => {ns => ''},
  38 );
  39
  40
  41 my $log = 'OpenSRF::Utils::Logger';
  42
  43 my  $parser = XML::LibXML->new();
  44 my  $xslt = XML::LibXSLT->new();
  45
  46 my  $mods_sheet;
  47 my  $mads_sheet;
  48 my  $xpathset = {};
  49 sub initialize {}
  50 sub child_init {}
  51
  52 sub post_init {
  53
  54         unless (keys %$xpathset) {
  55                 $log->debug("Running post_init", DEBUG);
  56
  57                 my $xsldir = OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl');
  58
  59                 unless ($supported_formats{mods}{xslt}) {
  60                         $log->debug("Loading MODS XSLT", DEBUG);
  61                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS.xsl");
  62                         $supported_formats{mods}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  63                 }
  64
  65                 unless ($supported_formats{mods3}{xslt}) {
  66                         $log->debug("Loading MODS v3 XSLT", DEBUG);
  67                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS3.xsl");
  68                         $supported_formats{mods3}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  69                 }
  70
  71                 unless ($supported_formats{mods32}{xslt}) {
  72                         $log->debug("Loading MODS v32 XSLT", DEBUG);
  73                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS32.xsl");
  74                         $supported_formats{mods32}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  75                 }
  76
  77                 my $req = OpenSRF::AppSession
  78                                 ->create('open-ils.cstore')
  79
  80                                 # XXX testing new metabib field use for faceting
  81                                 #->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { id => { '!=' => undef } } )
  82                                 ->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { search_field => 't' } )
  83
  84                                 ->gather(1);
  85
  86                 if (ref $req and @$req) {
  87                         for my $f (@$req) {
  88                                 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
  89                                 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
  90                                 $xpathset->{ $f->field_class }->{ $f->name }->{format} = $f->format;
  91                                 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
  92                         }
  93                 }
  94         }
  95 }
  96
  97 sub entityize {
  98         my $stuff = shift;
  99         my $form = shift;
 100
 101         if ($form eq 'D') {
 102                 $stuff = NFD($stuff);
 103         } else {
 104                 $stuff = NFC($stuff);
 105         }
 106
 107         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
 108         return $stuff;
 109 }
 110
 111 # --------------------------------------------------------------------------------
 112 # Biblio ingest
 113
 114 package OpenILS::Application::Ingest::Biblio;
 115 use base qw/OpenILS::Application::Ingest/;
 116 use Unicode::Normalize;
 117
 118 sub rw_biblio_ingest_single_object {
 119         my $self = shift;
 120         my $client = shift;
 121         my $bib = shift;
 122
 123         my ($blob) = $self->method_lookup("open-ils.ingest.full.biblio.object.readonly")->run($bib);
 124         return undef unless ($blob);
 125
 126         $bib->fingerprint( $blob->{fingerprint}->{fingerprint} );
 127         $bib->quality( $blob->{fingerprint}->{quality} );
 128
 129         my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
 130
 131         my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 132     my $tmp;
 133
 134         # update uri stuff ...
 135
 136     # gather URI call numbers for this record
 137     my $uri_cns = $u->{call_number} = $cstore->request(
 138         'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' }
 139     )->gather(1);
 140
 141     # gather the maps for those call numbers
 142     my $uri_maps = $u->{call_number} = $cstore->request(
 143         'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns }
 144     )->gather(1);
 145
 146     # delete the old maps
 147     $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps);
 148
 149     # and delete the call numbers if there are no more URIs
 150     if (!@{ $blob->{uri} }) {
 151         $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns);
 152     }
 153
 154     # now, add CNs, URIs and maps
 155     my %new_cns_by_owner;
 156     my %new_uris_by_owner;
 157     for my $u ( @{ $blob->{uri} } ) {
 158
 159         my $owner = $u->{call_number}->owning_lib;
 160
 161         if ($u->{call_number}->isnew) {
 162             if ($new_cns_by_owner{$owner}) {
 163                 $u->{call_number} = $new_cns_by_owner{$owner};
 164             } else {
 165                 $u->{call_number}->clear_id;
 166                 $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request(
 167                     'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number}
 168                 )->gather(1);
 169             }
 170         }
 171
 172         if ($u->{uri}->isnew) {
 173             if ($new_uris_by_owner{$owner}) {
 174                     $u->{uri} = $new_uris_by_owner{$owner};
 175             } else {
 176                     $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request(
 177                     'open-ils.cstore.direct.asset.uri.create' => $u->{uri}
 178                 )->gather(1);
 179             }
 180         }
 181
 182         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 183         $umap->uri($u->{uri}->id);
 184         $umap->call_number($u->{call_number}->id);
 185
 186             $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp);
 187     }
 188
 189         # update full_rec stuff ...
 190         $tmp = $cstore->request(
 191                 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic',
 192                 { record => $bib->id }
 193         )->gather(1);
 194
 195         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.delete' => $_ )->gather(1) for (@$tmp);
 196         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.create' => $_ )->gather(1) for (@{ $blob->{full_rec} });
 197
 198         # update rec_descriptor stuff ...
 199         $tmp = $cstore->request(
 200                 'open-ils.cstore.direct.metabib.record_descriptor.id_list.atomic',
 201                 { record => $bib->id }
 202         )->gather(1);
 203
 204         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.delete' => $_ )->gather(1) for (@$tmp);
 205         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.create' => $blob->{descriptor} )->gather(1);
 206
 207         # deal with classed fields...
 208         for my $class ( qw/title author subject keyword series/ ) {
 209                 $tmp = $cstore->request(
 210                         "open-ils.cstore.direct.metabib.${class}_field_entry.id_list.atomic",
 211                         { source => $bib->id }
 212                 )->gather(1);
 213
 214                 $cstore->request( "open-ils.cstore.direct.metabib.${class}_field_entry.delete" => $_ )->gather(1) for (@$tmp);
 215         }
 216         for my $obj ( @{ $blob->{field_entries} } ) {
 217                 my $class = $obj->class_name;
 218                 $class =~ s/^Fieldmapper:://o;
 219                 $class =~ s/::/./go;
 220                 $cstore->request( "open-ils.cstore.direct.$class.create" => $obj )->gather(1);
 221         }
 222
 223         # update MR map ...
 224
 225         $tmp = $cstore->request(
 226                 'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 227                 { source => $bib->id }
 228         )->gather(1);
 229
 230         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.delete' => $_->id )->gather(1) for (@$tmp);
 231
 232         # get the old MRs
 233         my $old_mrs = $cstore->request(
 234                 'open-ils.cstore.direct.metabib.metarecord.search.atomic' => { id => [map { $_->metarecord } @$tmp] }
 235         )->gather(1) if (@$tmp);
 236
 237         $old_mrs = [] if (!ref($old_mrs));
 238
 239         my $mr;
 240         for my $m (@$old_mrs) {
 241                 if ($m->fingerprint eq $bib->fingerprint) {
 242                         $mr = $m;
 243                 } else {
 244                         my $others = $cstore->request(
 245                                 'open-ils.cstore.direct.metabib.metarecord_source_map.id_list.atomic' => { metarecord => $m->id }
 246                         )->gather(1);
 247
 248                         if (!@$others) {
 249                                 $cstore->request(
 250                                         'open-ils.cstore.direct.metabib.metarecord.delete' => $m->id
 251                                 )->gather(1);
 252                         }
 253
 254                         $m->isdeleted(1);
 255                 }
 256         }
 257
 258         my $holds;
 259         if (!$mr) {
 260                 # Get the matchin MR, if any.
 261                 $mr = $cstore->request(
 262                         'open-ils.cstore.direct.metabib.metarecord.search',
 263                         { fingerprint => $bib->fingerprint }
 264                 )->gather(1);
 265
 266                 $holds = $cstore->request(
 267                         'open-ils.cstore.direct.action.hold_request.search.atomic',
 268                         { hold_type => 'M', target => [ map { $_->id } grep { $_->isdeleted } @$old_mrs ] }
 269                 )->gather(1) if (@$old_mrs);
 270
 271                 if ($mr) {
 272                         for my $h (@$holds) {
 273                                 $h->target($mr);
 274                                 $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 275                                 $h->ischanged(1);
 276                         }
 277                 }
 278         }
 279
 280         if (!$mr) {
 281                 $mr = new Fieldmapper::metabib::metarecord;
 282                 $mr->fingerprint( $bib->fingerprint );
 283                 $mr->master_record( $bib->id );
 284                 $mr->id(
 285                         $cstore->request(
 286                                 "open-ils.cstore.direct.metabib.metarecord.create",
 287                                 $mr => { quiet => 'true' }
 288                         )->gather(1)
 289                 );
 290
 291                 for my $h (grep { !$_->ischanged } @$holds) {
 292                         $h->target($mr);
 293                         $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 294                 }
 295         } else {
 296                 my $mrm = $cstore->request(
 297                         'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 298                         { metarecord => $mr->id }
 299                 )->gather(1);
 300
 301                 if (@$mrm) {
 302                         my $best = $cstore->request(
 303                                 "open-ils.cstore.direct.biblio.record_entry.search",
 304                                 { id => [ map { $_->source } @$mrm ] },
 305                                 { 'select'      => { bre => [ qw/id quality/ ] },
 306                                 order_by        => { bre => "quality desc" },
 307                                 limit           => 1,
 308                                 }
 309                         )->gather(1);
 310
 311                         if ($best->quality > $bib->quality) {
 312                                 $mr->master_record($best->id);
 313                         } else {
 314                                 $mr->master_record($bib->id);
 315                         }
 316                 } else {
 317                         $mr->master_record($bib->id);
 318                 }
 319
 320                 $mr->clear_mods;
 321
 322                 $cstore->request( 'open-ils.cstore.direct.metabib.metarecord.update' => $mr )->gather(1);
 323         }
 324
 325         my $mrm = new Fieldmapper::metabib::metarecord_source_map;
 326         $mrm->source($bib->id);
 327         $mrm->metarecord($mr->id);
 328
 329         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.create' => $mrm )->gather(1);
 330         $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.update' => $bib )->gather(1);
 331
 332         $cstore->request( 'open-ils.cstore.transaction.commit' )->gather(1) || return undef;;
 333     $cstore->disconnect;
 334
 335         return $bib->id;
 336 }
 337 __PACKAGE__->register_method(
 338         api_name        => "open-ils.ingest.full.biblio.object",
 339         method          => "rw_biblio_ingest_single_object",
 340         api_level       => 1,
 341         argc            => 1,
 342 );
 343
 344 sub rw_biblio_ingest_single_record {
 345         my $self = shift;
 346         my $client = shift;
 347         my $rec = shift;
 348
 349         OpenILS::Application::Ingest->post_init();
 350         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 351         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 352
 353         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
 354
 355         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 356         $cstore->disconnect;
 357
 358         return undef unless ($r and @$r);
 359
 360         return ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($r))[0];
 361 }
 362 __PACKAGE__->register_method(
 363         api_name        => "open-ils.ingest.full.biblio.record",
 364         method          => "rw_biblio_ingest_single_record",
 365         api_level       => 1,
 366         argc            => 1,
 367 );
 368
 369 sub rw_biblio_ingest_record_list {
 370         my $self = shift;
 371         my $client = shift;
 372         my @rec = ref($_[0]) ? @{ $_[0] } : @_ ;
 373
 374         OpenILS::Application::Ingest->post_init();
 375         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 376         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 377
 378         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.search.atomic' => { id => $rec } )->gather(1);
 379
 380         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 381         $cstore->disconnect;
 382
 383         return undef unless ($r and @$r);
 384
 385         my $count = 0;
 386         $count += ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($_))[0] for (@$r);
 387
 388         return $count;
 389 }
 390 __PACKAGE__->register_method(
 391         api_name        => "open-ils.ingest.full.biblio.record_list",
 392         method          => "rw_biblio_ingest_record_list",
 393         api_level       => 1,
 394         argc            => 1,
 395 );
 396
 397 sub ro_biblio_ingest_single_object {
 398         my $self = shift;
 399         my $client = shift;
 400         my $bib = shift;
 401         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 402     my $max_cn = shift;
 403     my $max_uri = shift;
 404
 405         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 406
 407     if (!$max_cn) {
 408         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef } }, { limit => 1, order_by => { acn => 'id desc' } } )->gather(1);
 409         $max_cn = int($cn->id) + 1000;
 410     }
 411
 412     if (!$max_uri) {
 413         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef } }, { limit => 1, order_by => { acn => 'id desc' } } )->gather(1);
 414         $max_uri = int($cn->id) + 1000;
 415     }
 416
 417     $cstore->disconnect;
 418
 419         my $document = $parser->parse_string($xml);
 420
 421         my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib, $max_cn, $max_uri);
 422         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 423         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 424         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 425         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 426
 427         $_->source($bib->id) for (@mXfe);
 428         $_->record($bib->id) for (@mfr);
 429         $rd->record($bib->id) if ($rd);
 430
 431         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris };
 432 }
 433 __PACKAGE__->register_method(
 434         api_name        => "open-ils.ingest.full.biblio.object.readonly",
 435         method          => "ro_biblio_ingest_single_object",
 436         api_level       => 1,
 437         argc            => 1,
 438 );
 439
 440 sub ro_biblio_ingest_single_xml {
 441         my $self = shift;
 442         my $client = shift;
 443         my $xml = OpenILS::Application::Ingest::entityize(shift);
 444
 445         my $document = $parser->parse_string($xml);
 446
 447         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 448         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 449         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 450         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 451
 452         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd };
 453 }
 454 __PACKAGE__->register_method(
 455         api_name        => "open-ils.ingest.full.biblio.xml.readonly",
 456         method          => "ro_biblio_ingest_single_xml",
 457         api_level       => 1,
 458         argc            => 1,
 459 );
 460
 461 sub ro_biblio_ingest_single_record {
 462         my $self = shift;
 463         my $client = shift;
 464         my $rec = shift;
 465
 466         OpenILS::Application::Ingest->post_init();
 467         my $r = OpenSRF::AppSession
 468                         ->create('open-ils.cstore')
 469                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 470                         ->gather(1);
 471
 472         return undef unless ($r and @$r);
 473
 474         my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($r->marc);
 475
 476         $_->source($rec) for (@{$res->{field_entries}});
 477         $_->record($rec) for (@{$res->{full_rec}});
 478         $res->{descriptor}->record($rec);
 479
 480         return $res;
 481 }
 482 __PACKAGE__->register_method(
 483         api_name        => "open-ils.ingest.full.biblio.record.readonly",
 484         method          => "ro_biblio_ingest_single_record",
 485         api_level       => 1,
 486         argc            => 1,
 487 );
 488
 489 sub ro_biblio_ingest_stream_record {
 490         my $self = shift;
 491         my $client = shift;
 492
 493         OpenILS::Application::Ingest->post_init();
 494
 495         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 496
 497         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 498
 499                 my $rec = $resp->content;
 500                 last unless (defined $rec);
 501
 502                 $log->debug("Running open-ils.ingest.full.biblio.record.readonly ...");
 503                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.record.readonly")->run($rec);
 504
 505                 $_->source($rec) for (@{$res->{field_entries}});
 506                 $_->record($rec) for (@{$res->{full_rec}});
 507
 508                 $client->respond( $res );
 509         }
 510
 511         return undef;
 512 }
 513 __PACKAGE__->register_method(
 514         api_name        => "open-ils.ingest.full.biblio.record_stream.readonly",
 515         method          => "ro_biblio_ingest_stream_record",
 516         api_level       => 1,
 517         stream          => 1,
 518 );
 519
 520 sub ro_biblio_ingest_stream_xml {
 521         my $self = shift;
 522         my $client = shift;
 523
 524         OpenILS::Application::Ingest->post_init();
 525
 526         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 527
 528         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 529
 530                 my $xml = $resp->content;
 531                 last unless (defined $xml);
 532
 533                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 534                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($xml);
 535
 536                 $client->respond( $res );
 537         }
 538
 539         return undef;
 540 }
 541 __PACKAGE__->register_method(
 542         api_name        => "open-ils.ingest.full.biblio.xml_stream.readonly",
 543         method          => "ro_biblio_ingest_stream_xml",
 544         api_level       => 1,
 545         stream          => 1,
 546 );
 547
 548 sub rw_biblio_ingest_stream_import {
 549         my $self = shift;
 550         my $client = shift;
 551
 552         OpenILS::Application::Ingest->post_init();
 553
 554         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 555
 556         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 557
 558                 my $bib = $resp->content;
 559                 last unless (defined $bib);
 560
 561                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 562                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($bib->marc);
 563
 564                 $_->source($bib->id) for (@{$res->{field_entries}});
 565                 $_->record($bib->id) for (@{$res->{full_rec}});
 566
 567                 $client->respond( $res );
 568         }
 569
 570         return undef;
 571 }
 572 __PACKAGE__->register_method(
 573         api_name        => "open-ils.ingest.full.biblio.bib_stream.import",
 574         method          => "rw_biblio_ingest_stream_import",
 575         api_level       => 1,
 576         stream          => 1,
 577 );
 578
 579
 580 # --------------------------------------------------------------------------------
 581 # Authority ingest
 582
 583 package OpenILS::Application::Ingest::Authority;
 584 use base qw/OpenILS::Application::Ingest/;
 585 use Unicode::Normalize;
 586
 587 sub ro_authority_ingest_single_object {
 588         my $self = shift;
 589         my $client = shift;
 590         my $bib = shift;
 591         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 592
 593         my $document = $parser->parse_string($xml);
 594
 595         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 596
 597         $_->record($bib->id) for (@mfr);
 598
 599         return { full_rec => \@mfr };
 600 }
 601 __PACKAGE__->register_method(
 602         api_name        => "open-ils.ingest.full.authority.object.readonly",
 603         method          => "ro_authority_ingest_single_object",
 604         api_level       => 1,
 605         argc            => 1,
 606 );
 607
 608 sub ro_authority_ingest_single_xml {
 609         my $self = shift;
 610         my $client = shift;
 611         my $xml = OpenILS::Application::Ingest::entityize(shift);
 612
 613         my $document = $parser->parse_string($xml);
 614
 615         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 616
 617         return { full_rec => \@mfr };
 618 }
 619 __PACKAGE__->register_method(
 620         api_name        => "open-ils.ingest.full.authority.xml.readonly",
 621         method          => "ro_authority_ingest_single_xml",
 622         api_level       => 1,
 623         argc            => 1,
 624 );
 625
 626 sub ro_authority_ingest_single_record {
 627         my $self = shift;
 628         my $client = shift;
 629         my $rec = shift;
 630
 631         OpenILS::Application::Ingest->post_init();
 632         my $r = OpenSRF::AppSession
 633                         ->create('open-ils.cstore')
 634                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 635                         ->gather(1);
 636
 637         return undef unless ($r and @$r);
 638
 639         my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($r->marc);
 640
 641         $_->record($rec) for (@{$res->{full_rec}});
 642         $res->{descriptor}->record($rec);
 643
 644         return $res;
 645 }
 646 __PACKAGE__->register_method(
 647         api_name        => "open-ils.ingest.full.authority.record.readonly",
 648         method          => "ro_authority_ingest_single_record",
 649         api_level       => 1,
 650         argc            => 1,
 651 );
 652
 653 sub ro_authority_ingest_stream_record {
 654         my $self = shift;
 655         my $client = shift;
 656
 657         OpenILS::Application::Ingest->post_init();
 658
 659         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 660
 661         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 662
 663                 my $rec = $resp->content;
 664                 last unless (defined $rec);
 665
 666                 $log->debug("Running open-ils.ingest.full.authority.record.readonly ...");
 667                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.record.readonly")->run($rec);
 668
 669                 $_->record($rec) for (@{$res->{full_rec}});
 670
 671                 $client->respond( $res );
 672         }
 673
 674         return undef;
 675 }
 676 __PACKAGE__->register_method(
 677         api_name        => "open-ils.ingest.full.authority.record_stream.readonly",
 678         method          => "ro_authority_ingest_stream_record",
 679         api_level       => 1,
 680         stream          => 1,
 681 );
 682
 683 sub ro_authority_ingest_stream_xml {
 684         my $self = shift;
 685         my $client = shift;
 686
 687         OpenILS::Application::Ingest->post_init();
 688
 689         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 690
 691         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 692
 693                 my $xml = $resp->content;
 694                 last unless (defined $xml);
 695
 696                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 697                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($xml);
 698
 699                 $client->respond( $res );
 700         }
 701
 702         return undef;
 703 }
 704 __PACKAGE__->register_method(
 705         api_name        => "open-ils.ingest.full.authority.xml_stream.readonly",
 706         method          => "ro_authority_ingest_stream_xml",
 707         api_level       => 1,
 708         stream          => 1,
 709 );
 710
 711 sub rw_authority_ingest_stream_import {
 712         my $self = shift;
 713         my $client = shift;
 714
 715         OpenILS::Application::Ingest->post_init();
 716
 717         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 718
 719         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 720
 721                 my $bib = $resp->content;
 722                 last unless (defined $bib);
 723
 724                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 725                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($bib->marc);
 726
 727                 $_->record($bib->id) for (@{$res->{full_rec}});
 728
 729                 $client->respond( $res );
 730         }
 731
 732         return undef;
 733 }
 734 __PACKAGE__->register_method(
 735         api_name        => "open-ils.ingest.full.authority.bib_stream.import",
 736         method          => "rw_authority_ingest_stream_import",
 737         api_level       => 1,
 738         stream          => 1,
 739 );
 740
 741
 742 # --------------------------------------------------------------------------------
 743 # MARC index extraction
 744
 745 package OpenILS::Application::Ingest::XPATH;
 746 use base qw/OpenILS::Application::Ingest/;
 747 use Unicode::Normalize;
 748
 749 # give this an XML documentElement and an XPATH expression
 750 sub xpath_to_string {
 751         my $xml = shift;
 752         my $xpath = shift;
 753         my $ns_uri = shift;
 754         my $ns_prefix = shift;
 755         my $unique = shift;
 756
 757         $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
 758
 759         my $string = "";
 760
 761         # grab the set of matching nodes
 762         my @nodes = $xml->findnodes( $xpath );
 763         for my $value (@nodes) {
 764
 765                 # grab all children of the node
 766                 my @children = $value->childNodes();
 767                 for my $child (@children) {
 768
 769                         # add the childs content to the growing buffer
 770                         my $content = quotemeta($child->textContent);
 771                         next if ($unique && $string =~ /$content/);  # uniquify the values
 772                         $string .= $child->textContent . " ";
 773                 }
 774                 if( ! @children ) {
 775                         $string .= $value->textContent . " ";
 776                 }
 777         }
 778
 779     $string =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 780     $string =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 781
 782         return NFD($string);
 783 }
 784
 785 sub class_index_string_xml {
 786         my $self = shift;
 787         my $client = shift;
 788         my $xml = shift;
 789         my @classes = @_;
 790
 791         OpenILS::Application::Ingest->post_init();
 792         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
 793
 794         my %transform_cache;
 795
 796         for my $class (@classes) {
 797                 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
 798                 for my $type ( keys %{ $xpathset->{$class} } ) {
 799
 800                         my $def = $xpathset->{$class}->{$type};
 801                         my $sf = $OpenILS::Application::Ingest::supported_formats{$def->{format}};
 802
 803                         my $document = $xml;
 804
 805                         if ($sf->{xslt}) {
 806                                 $document = $transform_cache{$def->{format}} || $sf->{xslt}->transform($xml);
 807                                 $transform_cache{$def->{format}} = $document;
 808                         }
 809
 810                         my $value =  xpath_to_string(
 811                                         $document->documentElement      => $def->{xpath},
 812                                         $sf->{ns}                       => $def->{format},
 813                                         1
 814                         );
 815
 816                         next unless $value;
 817
 818                         $value = NFD($value);
 819                         $value =~ s/\pM+//sgo;
 820                         $value =~ s/\pC+//sgo;
 821                         $value =~ s/\W+$//sgo;
 822
 823                         $value =~ s/\b\.+\b//sgo;
 824                         $value = lc($value);
 825
 826                         my $fm = $class_constructor->new;
 827                         $fm->value( $value );
 828                         $fm->field( $xpathset->{$class}->{$type}->{id} );
 829                         $client->respond($fm);
 830                 }
 831         }
 832         return undef;
 833 }
 834 __PACKAGE__->register_method(
 835         api_name        => "open-ils.ingest.field_entry.class.xml",
 836         method          => "class_index_string_xml",
 837         api_level       => 1,
 838         argc            => 2,
 839         stream          => 1,
 840 );
 841
 842 sub class_index_string_record {
 843         my $self = shift;
 844         my $client = shift;
 845         my $rec = shift;
 846         my @classes = shift;
 847
 848         OpenILS::Application::Ingest->post_init();
 849         my $r = OpenSRF::AppSession
 850                         ->create('open-ils.cstore')
 851                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 852                         ->gather(1);
 853
 854         return undef unless ($r and @$r);
 855
 856         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, @classes)) {
 857                 $fm->source($rec);
 858                 $client->respond($fm);
 859         }
 860         return undef;
 861 }
 862 __PACKAGE__->register_method(
 863         api_name        => "open-ils.ingest.field_entry.class.record",
 864         method          => "class_index_string_record",
 865         api_level       => 1,
 866         argc            => 2,
 867         stream          => 1,
 868 );
 869
 870 sub all_index_string_xml {
 871         my $self = shift;
 872         my $client = shift;
 873         my $xml = shift;
 874
 875         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($xml, keys(%$xpathset))) {
 876                 $client->respond($fm);
 877         }
 878         return undef;
 879 }
 880 __PACKAGE__->register_method(
 881         api_name        => "open-ils.ingest.extract.field_entry.all.xml",
 882         method          => "all_index_string_xml",
 883         api_level       => 1,
 884         argc            => 1,
 885         stream          => 1,
 886 );
 887
 888 sub all_index_string_record {
 889         my $self = shift;
 890         my $client = shift;
 891         my $rec = shift;
 892
 893         OpenILS::Application::Ingest->post_init();
 894         my $r = OpenSRF::AppSession
 895                         ->create('open-ils.cstore')
 896                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 897                         ->gather(1);
 898
 899         return undef unless ($r and @$r);
 900
 901         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, keys(%$xpathset))) {
 902                 $fm->source($rec);
 903                 $client->respond($fm);
 904         }
 905         return undef;
 906 }
 907 __PACKAGE__->register_method(
 908         api_name        => "open-ils.ingest.extract.field_entry.all.record",
 909         method          => "all_index_string_record",
 910         api_level       => 1,
 911         argc            => 1,
 912         stream          => 1,
 913 );
 914
 915 # --------------------------------------------------------------------------------
 916 # Flat MARC
 917
 918 package OpenILS::Application::Ingest::FlatMARC;
 919 use base qw/OpenILS::Application::Ingest/;
 920 use Unicode::Normalize;
 921
 922
 923 sub _marcxml_to_full_rows {
 924
 925         my $marcxml = shift;
 926         my $xmltype = shift || 'metabib';
 927
 928         my $type = "Fieldmapper::${xmltype}::full_rec";
 929
 930         my @ns_list;
 931
 932         my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
 933
 934         for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
 935                 next unless $tagline;
 936
 937                 my $ns = $type->new;
 938
 939                 $ns->tag( 'LDR' );
 940                 my $val = $tagline->textContent;
 941                 $val = NFD($val);
 942                 $val =~ s/\pM+//sgo;
 943                 $val =~ s/\pC+//sgo;
 944                 $val =~ s/\W+$//sgo;
 945                 $ns->value( $val );
 946
 947                 push @ns_list, $ns;
 948         }
 949
 950         for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
 951                 next unless $tagline;
 952
 953                 my $ns = $type->new;
 954
 955                 $ns->tag( $tagline->getAttribute( "tag" ) );
 956                 my $val = $tagline->textContent;
 957                 $val = NFD($val);
 958                 $val =~ s/\pM+//sgo;
 959                 $val =~ s/\pC+//sgo;
 960                 $val =~ s/\W+$//sgo;
 961                 $ns->value( $val );
 962
 963                 push @ns_list, $ns;
 964         }
 965
 966         for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
 967                 next unless $tagline;
 968
 969                 my $tag = $tagline->getAttribute( "tag" );
 970                 my $ind1 = $tagline->getAttribute( "ind1" );
 971                 my $ind2 = $tagline->getAttribute( "ind2" );
 972
 973                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 974                         next unless $data;
 975
 976                         my $ns = $type->new;
 977
 978                         $ns->tag( $tag );
 979                         $ns->ind1( $ind1 );
 980                         $ns->ind2( $ind2 );
 981                         $ns->subfield( $data->getAttribute( "code" ) );
 982                         my $val = $data->textContent;
 983                         $val = NFD($val);
 984                         $val =~ s/\pM+//sgo;
 985                         $val =~ s/\pC+//sgo;
 986                         $val =~ s/\W+$//sgo;
 987             $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 988             $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 989                         $ns->value( lc($val) );
 990
 991                         push @ns_list, $ns;
 992                 }
 993
 994         if ($xmltype eq 'metabib' and $tag eq '245') {
 995                 $tag = 'tnf';
 996
 997                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 998                         next unless ($data and $data->getAttribute( "code" ) eq 'a');
 999
1000                         $ns = $type->new;
1001
1002                         $ns->tag( $tag );
1003                         $ns->ind1( $ind1 );
1004                         $ns->ind2( $ind2 );
1005                         $ns->subfield( $data->getAttribute( "code" ) );
1006                         my $val = substr( $data->textContent, $ind2 );
1007                         $val = NFD($val);
1008                         $val =~ s/\pM+//sgo;
1009                         $val =~ s/\pC+//sgo;
1010                         $val =~ s/\W+$//sgo;
1011                 $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
1012                 $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
1013                         $ns->value( lc($val) );
1014
1015                         push @ns_list, $ns;
1016                 }
1017         }
1018         }
1019
1020         $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml");
1021         return @ns_list;
1022 }
1023
1024 sub flat_marc_xml {
1025         my $self = shift;
1026         my $client = shift;
1027         my $xml = shift;
1028
1029         $log->debug("processing [$xml]");
1030
1031         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
1032
1033         my $type = 'metabib';
1034         $type = 'authority' if ($self->api_name =~ /authority/o);
1035
1036         OpenILS::Application::Ingest->post_init();
1037
1038         $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
1039         return undef;
1040 }
1041 __PACKAGE__->register_method(
1042         api_name        => "open-ils.ingest.flat_marc.authority.xml",
1043         method          => "flat_marc_xml",
1044         api_level       => 1,
1045         argc            => 1,
1046         stream          => 1,
1047 );
1048 __PACKAGE__->register_method(
1049         api_name        => "open-ils.ingest.flat_marc.biblio.xml",
1050         method          => "flat_marc_xml",
1051         api_level       => 1,
1052         argc            => 1,
1053         stream          => 1,
1054 );
1055
1056 sub flat_marc_record {
1057         my $self = shift;
1058         my $client = shift;
1059         my $rec = shift;
1060
1061         my $type = 'biblio';
1062         $type = 'authority' if ($self->api_name =~ /authority/o);
1063
1064         OpenILS::Application::Ingest->post_init();
1065         my $r = OpenSRF::AppSession
1066                         ->create('open-ils.cstore')
1067                         ->request( "open-ils.cstore.direct.${type}.record_entry.retrieve" => $rec )
1068                         ->gather(1);
1069
1070
1071         return undef unless ($r and $r->marc);
1072
1073         my @rows = $self->method_lookup("open-ils.ingest.flat_marc.$type.xml")->run($r->marc);
1074         for my $row (@rows) {
1075                 $client->respond($row);
1076                 $log->debug(OpenSRF::Utils::JSON->perl2JSON($row), DEBUG);
1077         }
1078         return undef;
1079 }
1080 __PACKAGE__->register_method(
1081         api_name        => "open-ils.ingest.flat_marc.biblio.record_entry",
1082         method          => "flat_marc_record",
1083         api_level       => 1,
1084         argc            => 1,
1085         stream          => 1,
1086 );
1087 __PACKAGE__->register_method(
1088         api_name        => "open-ils.ingest.flat_marc.authority.record_entry",
1089         method          => "flat_marc_record",
1090         api_level       => 1,
1091         argc            => 1,
1092         stream          => 1,
1093 );
1094
1095
1096 # --------------------------------------------------------------------------------
1097 # URI extraction
1098
1099 package OpenILS::Application::Ingest::Biblio::URI;
1100 use base qw/OpenILS::Application::Ingest/;
1101 use Unicode::Normalize;
1102 use OpenSRF::EX qw/:try/;
1103
1104
1105 sub _extract_856_uris {
1106
1107     my $rec   = shift;
1108         my $max_cn = shift;
1109         my $max_uri = shift;
1110         my @objects;
1111
1112     my $recid = $rec->id;
1113     my $marcxml = $rec->marc;
1114
1115         my $document = $parser->parse_string($marcxml);
1116         my @nodes = $document->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]');
1117
1118     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
1119
1120     for my $node (@nodes) {
1121         # first, is there a URI?
1122         my $href = $node->findvalue('*[local-name()="subfield" and @code="u"]/text()');
1123         next unless ($href);
1124
1125         # now, find the best possible label
1126         my $label = $node->findvalue('*[local-name()="subfield" and @code="y"]/text()');
1127         $label ||= $node->findvalue('*[local-name()="subfield" and @code="3"]/text()');
1128         $label ||= $href;
1129
1130         # look for use info
1131         my $use = $node->findvalue('*[local-name()="subfield" and @code="z"]/text()');
1132         $use ||= $node->findvalue('*[local-name()="subfield" and @code="2"]/text()');
1133         $use ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1134
1135         # moving on to the URI owner
1136         my $owner = $node->findvalue('*[local-name()="subfield" and @code="w"]/text()');
1137         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1138         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce
1139
1140         $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ...
1141
1142         # no owner? skip it :(
1143         next unless ($owner);
1144
1145         my $org = $cstore
1146             ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} )
1147                         ->gather(1);
1148
1149         next unless ($org);
1150
1151         # now we can construct the uri object
1152         my $uri = $cstore
1153             ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use_restriction => $use, active => 't' } )
1154                         ->gather(1);
1155
1156         if (!$uri) {
1157             $uri = Fieldmapper::asset::uri->new;
1158             $uri->isnew( 1 );
1159             $uri->id( $$max_uri++ );
1160             $uri->label($label);
1161             $uri->href($href);
1162             $uri->active('t');
1163             $uri->use_restriction($use);
1164         }
1165
1166         # see if we need to create a call number
1167         my $cn = $cstore
1168             ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owning_lib => $org->id, record => $recid, label => '##URI##' } )
1169                         ->gather(1);
1170
1171         if (!$cn) {
1172             $c = Fieldmapper::asset::call_number->new;
1173             $cn->isnew( 1 );
1174             $cn->deleted('f');
1175             $cn->id( $$max_cn++ );
1176             $cn->owning_lib( $org->id );
1177             $cn->record( $recid );
1178             $cn->create_date( 'now' );
1179             $cn->creator( $rec->creator );
1180             $cn->editor( $rec->editor );
1181             $cn->edit_date( 'now' );
1182             $cn->label( '##URI##' );
1183         }
1184
1185         push @objects, { uri => $uri, call_number => $cn };
1186     }
1187
1188         $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid");
1189         return @objects;
1190 }
1191
1192 sub get_uris_record {
1193         my $self = shift;
1194         my $client = shift;
1195         my $rec = shift;
1196
1197         OpenILS::Application::Ingest->post_init();
1198         my $r = OpenSRF::AppSession
1199                         ->create('open-ils.cstore')
1200                         ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec )
1201                         ->gather(1);
1202
1203         return undef unless ($r and $r->marc);
1204
1205         $client->respond($_) for (_extract_856_uris($r));
1206         return undef;
1207 }
1208 __PACKAGE__->register_method(
1209         api_name        => "open-ils.ingest.856_uri.record",
1210         method          => "get_uris_record",
1211         api_level       => 1,
1212         argc            => 1,
1213         stream          => 1,
1214 );
1215
1216 sub get_uris_object {
1217         my $self = shift;
1218         my $client = shift;
1219         my $obj = shift;
1220         my $max_cn = shift;
1221         my $max_uri = shift;
1222
1223         return undef unless ($obj and $obj->marc);
1224
1225         $client->respond($_) for (_extract_856_uris($obj, \$max_cn, \$max_uri));
1226         return undef;
1227 }
1228 __PACKAGE__->register_method(
1229         api_name        => "open-ils.ingest.856_uri.object",
1230         method          => "get_uris_object",
1231         api_level       => 1,
1232         argc            => 1,
1233         stream          => 1,
1234 );
1235
1236
1237 # --------------------------------------------------------------------------------
1238 # Fingerprinting
1239
1240 package OpenILS::Application::Ingest::Biblio::Fingerprint;
1241 use base qw/OpenILS::Application::Ingest/;
1242 use Unicode::Normalize;
1243 use OpenSRF::EX qw/:try/;
1244
1245 sub biblio_fingerprint_record {
1246         my $self = shift;
1247         my $client = shift;
1248         my $rec = shift;
1249
1250         OpenILS::Application::Ingest->post_init();
1251
1252         my $r = OpenSRF::AppSession
1253                         ->create('open-ils.cstore')
1254                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
1255                         ->gather(1);
1256
1257         return undef unless ($r and $r->marc);
1258
1259         my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
1260         $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
1261         $fp->{quality} = int($fp->{quality});
1262         return $fp;
1263 }
1264 __PACKAGE__->register_method(
1265         api_name        => "open-ils.ingest.fingerprint.record",
1266         method          => "biblio_fingerprint_record",
1267         api_level       => 1,
1268         argc            => 1,
1269 );
1270
1271 our $fp_script;
1272 sub biblio_fingerprint {
1273         my $self = shift;
1274         my $client = shift;
1275         my $xml = OpenILS::Application::Ingest::entityize(shift);
1276
1277         $log->internal("Got MARC [$xml]");
1278
1279         if(!$fp_script) {
1280                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1281                 my $conf = OpenSRF::Utils::SettingsClient->new;
1282
1283                 my $libs        = $conf->config_value(@pfx, 'script_path');
1284                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_fingerprint');
1285                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1286
1287                 $log->debug("Loading script $script_file for biblio fingerprinting...");
1288
1289                 $fp_script = new OpenILS::Utils::ScriptRunner
1290                         ( file          => $script_file,
1291                           paths         => $script_libs,
1292                           reset_count   => 100 );
1293         }
1294
1295         $fp_script->insert('environment' => {marc => $xml} => 1);
1296
1297         my $res = $fp_script->run || ($log->error( "Fingerprint script died!  $@" ) && return undef);
1298         $log->debug("Script for biblio fingerprinting completed successfully...");
1299
1300         return $res;
1301 }
1302 __PACKAGE__->register_method(
1303         api_name        => "open-ils.ingest.fingerprint.xml",
1304         method          => "biblio_fingerprint",
1305         api_level       => 1,
1306         argc            => 1,
1307 );
1308
1309 our $rd_script;
1310 sub biblio_descriptor {
1311         my $self = shift;
1312         my $client = shift;
1313         my $xml = OpenILS::Application::Ingest::entityize(shift);
1314
1315         $log->internal("Got MARC [$xml]");
1316
1317         if(!$rd_script) {
1318                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1319                 my $conf = OpenSRF::Utils::SettingsClient->new;
1320
1321                 my $libs        = $conf->config_value(@pfx, 'script_path');
1322                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_descriptor');
1323                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1324
1325                 $log->debug("Loading script $script_file for biblio descriptor extraction...");
1326
1327                 $rd_script = new OpenILS::Utils::ScriptRunner
1328                         ( file          => $script_file,
1329                           paths         => $script_libs,
1330                           reset_count   => 100 );
1331         }
1332
1333         $log->debug("Setting up environment for descriptor extraction script...");
1334         $rd_script->insert('environment.marc' => $xml => 1);
1335         $log->debug("Environment building complete...");
1336
1337         my $res = $rd_script->run || ($log->error( "Descriptor script died!  $@" ) && return undef);
1338         $log->debug("Script for biblio descriptor extraction completed successfully");
1339
1340     my $d1 = $res->date1;
1341     if ($d1 && $d1 ne '    ') {
1342         $d1 =~ tr/ux/00/;
1343         $res->date1( $d1 );
1344     }
1345
1346     my $d2 = $res->date2;
1347     if ($d2 && $d2 ne '    ') {
1348         $d2 =~ tr/ux/99/;
1349         $res->date2( $d2 );
1350     }
1351
1352         return $res;
1353 }
1354 __PACKAGE__->register_method(
1355         api_name        => "open-ils.ingest.descriptor.xml",
1356         method          => "biblio_descriptor",
1357         api_level       => 1,
1358         argc            => 1,
1359 );
1360
1361
1362 1;
1363