From 71b46babe2c116f29ea0938d258139e535dc04bc Mon Sep 17 00:00:00 2001 From: miker Date: Tue, 12 Apr 2005 14:10:33 +0000 Subject: [PATCH] big changes in little china git-svn-id: svn://svn.open-ils.org/ILS/trunk@476 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- .../perlmods/OpenILS/Application/Storage.pm | 12 +- .../OpenILS/Application/Storage/CDBI.pm | 20 ++-- .../OpenILS/Application/Storage/CDBI/actor.pm | 20 ++-- .../OpenILS/Application/Storage/CDBI/asset.pm | 14 +-- .../Application/Storage/CDBI/biblio.pm | 1 - .../Application/Storage/CDBI/config.pm | 27 +++++ .../OpenILS/Application/Storage/CDBI/money.pm | 18 ++- .../OpenILS/Application/Storage/Driver/Pg.pm | 19 ++- .../OpenILS/Application/Storage/FTS.pm | 4 +- .../OpenILS/Application/Storage/Publisher.pm | 2 +- .../Application/Storage/Publisher/metabib.pm | 111 ++++++++++++------ .../OpenILS/Application/Storage/WORM.pm | 61 ++++++++-- .../src/sql/Postgres/002.schema.config.sql | 22 ++-- .../src/sql/Postgres/005.schema.actors.sql | 10 +- 14 files changed, 240 insertions(+), 101 deletions(-) diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage.pm index 8667999147..11a6b9d65b 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage.pm @@ -7,12 +7,8 @@ use OpenSRF::Utils::Logger qw/:level/; # Pull this in so we can adjust it's @ISA use OpenILS::Application::Storage::CDBI (1); - use OpenILS::Application::Storage::FTS; -# Suck in the method publishing modules -use OpenILS::Application::Storage::Publisher; -use OpenILS::Application::Storage::WORM; # the easy way to get to the logger... my $log = "OpenSRF::Utils::Logger"; @@ -40,7 +36,11 @@ sub initialize { $log->debug("$driver loaded successfully", DEBUG); + # Suck in the method publishing modules @OpenILS::Application::Storage::CDBI::ISA = ( $driver ); + + eval 'use OpenILS::Application::Storage::Publisher;'; + eval 'use OpenILS::Application::Storage::WORM;'; } sub child_init { @@ -49,10 +49,6 @@ sub child_init { my $conf = OpenSRF::Utils::SettingsClient->new; - # XXX -- this died... router down? - #$log->debug('Prepopulating method_lookup cache', DEBUG); - #OpenSRF::Application->method_lookup('crappola'); - $log->debug('Calling the Driver child_init', DEBUG); OpenILS::Application::Storage::CDBI->child_init( $conf->config_value( apps => 'open-ils.storage' => app_settings => databases => 'database') diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI.pm index d67292f796..1744a87ef2 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI.pm @@ -2,15 +2,16 @@ package OpenILS::Application::Storage::CDBI; use base qw/Class::DBI/; use Class::DBI; -use OpenILS::Application::Storage::CDBI::config; use OpenILS::Application::Storage::CDBI::actor; use OpenILS::Application::Storage::CDBI::action; use OpenILS::Application::Storage::CDBI::asset; use OpenILS::Application::Storage::CDBI::biblio; +use OpenILS::Application::Storage::CDBI::config; use OpenILS::Application::Storage::CDBI::metabib; use OpenILS::Application::Storage::CDBI::money; use OpenSRF::Utils::Logger; +use OpenSRF::EX qw/:try/; our $VERSION; my $log = 'OpenSRF::Utils::Logger'; @@ -120,11 +121,13 @@ sub retrieve { $arg = $arg->id; } $log->debug("Retrieving $self with $arg", INTERNAL); - my $rec = $self->SUPER::retrieve("$arg"); - unless ($rec) { - $log->debug("Could not retrieve $self with $arg!", DEBUG); + my $rec; + try { + $rec = $self->SUPER::retrieve("$arg"); + } catch Error with { + $log->debug("Could not retrieve $self with $arg! -- ".shift(), DEBUG); return undef; - } + }; return $rec; } @@ -165,12 +168,13 @@ sub create_from_fieldmapper { $log->debug("Creating node of type ".ref($fm), DEBUG); my $class = ref($obj) || $obj; + my $primary = $class->primary_column; if (ref $fm) { my %hash = map { defined $fm->$_ ? ($_ => $fm->$_) : () - } $fm->real_fields; + } grep { $_ ne $primary } $class->columns('All'); if ($class->find_column( 'last_xact_id' )) { my $xact_id = $class->current_xact_id; @@ -237,10 +241,9 @@ sub modify_from_fieldmapper { if (!ref($obj)) { $obj = $class->retrieve($fm); unless ($obj) { - $log->debug("Rretrieve using $fm (".$fm->id.") failed!", ERROR); + $log->debug("Retrieve of $class using $fm (".$fm->id.") failed! -- ".shift(), ERROR); throw OpenSRF::EX::WARN ("No $class with id of ".$fm->id."!!"); } - } my %hash = map { defined $fm->$_ ? @@ -271,6 +274,7 @@ sub modify_from_fieldmapper { sub import { return if ($VERSION); + #------------------------------------------------------------------------------- actor::user->has_a( home_ou => 'actor::org_unit' ); actor::user->has_many( survey_responses => 'action::survey_response' ); diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/actor.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/actor.pm index 886d0f49c1..cca7a72bde 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/actor.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/actor.pm @@ -10,10 +10,12 @@ use base qw/actor/; __PACKAGE__->table( 'actor_usr' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/id usrid usrname email prefix first_given_name - second_given_name family_name suffix address +__PACKAGE__->columns( Essential => qw/usrid usrname email first_given_name + second_given_name family_name address home_ou gender dob active master_account - super_user usrgroup passwd last_xact_id/ ); + ident_type ident_value super_user usrgroup + passwd last_xact_id/ ); +__PACKAGE__->columns( Others => qw/prefix suffix address/ ); #------------------------------------------------------------------------------- package actor::org_unit_type; @@ -21,7 +23,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_org_unit_type' ); __PACKAGE__->columns( Primary => qw/id/); -__PACKAGE__->columns( Others => qw/name depth parent can_have_users/); +__PACKAGE__->columns( Essential => qw/name depth parent can_have_users/); #------------------------------------------------------------------------------- package actor::org_unit; @@ -29,7 +31,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_org_unit' ); __PACKAGE__->columns( Primary => qw/id/); -__PACKAGE__->columns( Others => qw/parent_ou ou_type address shortname name/); +__PACKAGE__->columns( Essential => qw/parent_ou ou_type address shortname name/); #------------------------------------------------------------------------------- package actor::stat_cat; @@ -37,7 +39,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_stat_cat' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owner name opac_visible/ ); +__PACKAGE__->columns( Essential => qw/owner name opac_visible/ ); #------------------------------------------------------------------------------- package actor::stat_cat_entry; @@ -45,7 +47,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_stat_cat_entry' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owner value/ ); +__PACKAGE__->columns( Essential => qw/owner value/ ); #------------------------------------------------------------------------------- package actor::stat_cat_entry_user_map; @@ -53,7 +55,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_stat_cat_entry_usr_map' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/stat_cat_entry target_user/ ); +__PACKAGE__->columns( Essential => qw/stat_cat_entry target_user/ ); #------------------------------------------------------------------------------- package actor::card; @@ -61,7 +63,7 @@ use base qw/actor/; __PACKAGE__->table( 'actor_card' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/usr barcode active/ ); +__PACKAGE__->columns( Essential => qw/usr barcode active/ ); #------------------------------------------------------------------------------- package actor::user_access_entry; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/asset.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/asset.pm index 10a161c599..9890865089 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/asset.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/asset.pm @@ -10,7 +10,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_call_number' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/record label creator create_date editor edit_date record label owning_lib/ ); +__PACKAGE__->columns( Essential => qw/record label creator create_date editor edit_date record label owning_lib/ ); #------------------------------------------------------------------------------- package asset::call_number_note; @@ -18,7 +18,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_call_number_note' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owning_call_number title creator create_date value/ ); +__PACKAGE__->columns( Essential => qw/owning_call_number title creator create_date value/ ); #------------------------------------------------------------------------------- package asset::copy; @@ -26,7 +26,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_copy' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/call_number barcode creator create_date editor +__PACKAGE__->columns( Essential => qw/call_number barcode creator create_date editor edit_date copy_number available loan_duration fine_level circulate deposit price ref opac_visible circ_as_type circ_modifier deposit_amount/ ); @@ -37,7 +37,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_stat_cat' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owner name opac_visible/ ); +__PACKAGE__->columns( Essential => qw/owner name opac_visible/ ); #------------------------------------------------------------------------------- package asset::stat_cat_entry; @@ -45,7 +45,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_stat_cat_entry' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owner value/ ); +__PACKAGE__->columns( Essential => qw/owner value/ ); #------------------------------------------------------------------------------- package asset::stat_cat_entry_copy_map; @@ -53,7 +53,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_stat_cat_entry_copy_map' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/stat_cat stat_cat_entry owning_copy/ ); +__PACKAGE__->columns( Essential => qw/stat_cat stat_cat_entry owning_copy/ ); #------------------------------------------------------------------------------- package asset::copy_note; @@ -61,7 +61,7 @@ use base qw/asset/; __PACKAGE__->table( 'asset_copy_note' ); __PACKAGE__->columns( Primary => qw/id/ ); -__PACKAGE__->columns( Others => qw/owning_copy title creator create_date value/ ); +__PACKAGE__->columns( Essential => qw/owning_copy title creator create_date value/ ); #------------------------------------------------------------------------------- diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/biblio.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/biblio.pm index 185e9c1f66..14a2ca4469 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/biblio.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/biblio.pm @@ -7,7 +7,6 @@ use base qw/OpenILS::Application::Storage::CDBI/; #------------------------------------------------------------------------------- package biblio::record_entry; use base qw/biblio/; -#use OpenILS::Application::Storage::CDBI::asset; biblio::record_entry->table( 'biblio_record_entry' ); biblio::record_entry->columns( Primary => 'id' ); diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/config.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/config.pm index 45a9e39c00..68144afcac 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/config.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/config.pm @@ -27,6 +27,33 @@ __PACKAGE__->columns(Primary => 'id'); __PACKAGE__->columns(Essential => qw/name/); #------------------------------------------------------------------------------- +package config::rules::circ_duration; +use base qw/config/; +__PACKAGE__->table('config_rule_circ_duration'); +__PACKAGE__->columns(Primary => 'id'); +__PACKAGE__->columns(Essential => qw/name extended normal short max_renewals/); +#------------------------------------------------------------------------------- + +package config::rules::max_fine; +use base qw/config/; +__PACKAGE__->table('config_rule_max_fine'); +__PACKAGE__->columns(Primary => 'id'); +__PACKAGE__->columns(Essential => qw/name amount/); +#------------------------------------------------------------------------------- + +package config::rules::recuring_fine; +use base qw/config/; +__PACKAGE__->table('config_rule_recuring_fine'); +__PACKAGE__->columns(Primary => 'id'); +__PACKAGE__->columns(Essential => qw/name high normal low/); +#------------------------------------------------------------------------------- + +package config::rules::age_hold_protect; +use base qw/config/; +__PACKAGE__->table('config_rule_age_hold_protect'); +__PACKAGE__->columns(Primary => 'id'); +__PACKAGE__->columns(Essential => qw/name age radius/); +#------------------------------------------------------------------------------- 1; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/money.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/money.pm index 2f71628a9a..f80062542c 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/money.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/CDBI/money.pm @@ -10,35 +10,40 @@ package money::billable_transaction; use base qw/money/; __PACKAGE__->table('money_billable_xact'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => qw/xact_start xact_finish usr/); +__PACKAGE__->columns(Essential => qw/xact_start usr/); +__PACKAGE__->columns(Others => qw/xact_finish/); #------------------------------------------------------------------------------- package money::billing; use base qw/money/; __PACKAGE__->table('money_billing'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => qw/xact amount billing_ts note/); +__PACKAGE__->columns(Essential => qw/xact amount billing_ts/); +__PACKAGE__->columns(Others => qw/note/); #------------------------------------------------------------------------------- package money::payment; use base qw/money/; __PACKAGE__->table('money_payment'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => "split_part(tableoid2name(tableoid),'.',2) as payment_type", qw/xact amount payment_ts note/); +__PACKAGE__->columns(Essential => qw/xact amount payment_ts/); +__PACKAGE__->columns(Others => qw/note/); #------------------------------------------------------------------------------- package money::cash_payment; use base qw/money/; __PACKAGE__->table('money_cash_payment'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => qw/xact amount payment_ts note cash_drawer accepting_usr amount_collected/); +__PACKAGE__->columns(Essential => qw/xact amount payment_ts cash_drawer accepting_usr amount_collected/); +__PACKAGE__->columns(Others => qw/note/); #------------------------------------------------------------------------------- package money::check_payment; use base qw/money/; __PACKAGE__->table('money_check_payment'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => qw/xact amount payment_ts note cash_drawer check_number accepting_usr amount_collected/); +__PACKAGE__->columns(Essential => qw/xact amount payment_ts cash_drawer check_number accepting_usr amount_collected/); +__PACKAGE__->columns(Others => qw/note/); #------------------------------------------------------------------------------- package money::credit_card_payment; @@ -56,7 +61,8 @@ package money::forgive_payment; use base qw/money/; __PACKAGE__->table('money_payment'); __PACKAGE__->columns(Primary => 'id'); -__PACKAGE__->columns(Essential => qw/xact amount payment_ts note accepting_usr amount_collected/); +__PACKAGE__->columns(Essential => qw/xact amount payment_ts accepting_usr amount_collected/); +__PACKAGE__->columns(Others => qw/note/); #------------------------------------------------------------------------------- package money::work_payment; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Driver/Pg.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Driver/Pg.pm index e74c8b6a0c..7c1be736f0 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Driver/Pg.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Driver/Pg.pm @@ -367,8 +367,6 @@ my $client = shift; my @fm_nodes = @_; - warn 'Inside copy_create...'; - return undef unless ($pg->current_xact_session); my $cdbi = $self->{cdbi}; @@ -551,6 +549,15 @@ metabib::metarecord->table( 'metabib.metarecord' ); metabib::metarecord->sequence( 'metabib.metarecord_id_seq' ); + OpenILS::Application::Storage->register_method( + api_name => 'open-ils.storage.direct.metabib.metarecord.batch.create', + method => 'copy_create', + api_level => 1, + 'package' => 'OpenILS::Application::Storage', + cdbi => 'metabib::metarecord', + ); + + #------------------------------------------------------------------------------- #------------------------------------------------------------------------------- @@ -660,6 +667,14 @@ package metabib::metarecord_source_map; metabib::metarecord_source_map->table( 'metabib.metarecord_source_map' ); + OpenILS::Application::Storage->register_method( + api_name => 'open-ils.storage.direct.metabib.metarecord_source_map.batch.create', + method => 'copy_create', + api_level => 1, + 'package' => 'OpenILS::Application::Storage', + cdbi => 'metabib::metarecord_source_map', + ); + #------------------------------------------------------------------------------- package metabib::record_descriptor; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/FTS.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/FTS.pm index a0040872ef..5693910e9a 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/FTS.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/FTS.pm @@ -5,7 +5,7 @@ my $log = 'OpenSRF::Utils::Logger'; package OpenILS::Application::Storage::FTS; use OpenSRF::Utils::Logger qw/:level/; -sub default_compile { +sub compile { $log->debug("You must override me somewhere, or I will make searching really slow!!!!",ERROR);; @@ -194,7 +194,7 @@ package Class::DBI; push @vals, ''.$class->_deflated_column($column, $v); } } else { - push @frags, "$cols[$_] $search_type $$search_opts{_placeholder}"; + push @frags, "$col $search_type $$search_opts{_placeholder}"; push @vals, $class->_deflated_column($column, $val); } } diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher.pm index 0c7d5dff65..02f824acfb 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher.pm @@ -7,7 +7,7 @@ use OpenSRF::Utils::Logger; my $log = 'OpenSRF::Utils::Logger'; use OpenILS::Utils::Fieldmapper; -use OpenILS::Application::Storage::CDBI; +#use OpenILS::Application::Storage::CDBI; #use OpenILS::Application::Storage::CDBI::actor; #use OpenILS::Application::Storage::CDBI::asset; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm index d1f85e2a67..217d9de1bc 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/Publisher/metabib.pm @@ -18,17 +18,28 @@ $VERSION = 1; sub search_full_rec { my $self = shift; my $client = shift; - my $limiters = shift; - my $term = shift; - my $window = shift; - $window = 100 unless (defined $window); - my $cache_key = md5_hex(Dump($limiters).$term); + my %args = @_; + + my $term = $args{term}; + my $limiters = $args{restrict}; + my $limit = $args{limit} || 100; + my $offset = $args{offset} || 0; + + + my $cache_key = md5_hex(Dumper($limiters).$term); + + my $cached_recs = OpenSRF::Utils::Cache->new->get_cache( $cache_key ); + return [ @$cached_recs[$offset .. $limit - 1] ] if (defined $cached_recs); - my ($fts_col) = metabib::full_rec->columns('FTS'); - my $table = metabib::full_rec->table; + my ($index_col) = metabib::full_rec->columns('FTS'); + $index_col ||= 'value'; + my $search_table = metabib::full_rec->table; - my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value','index_vector'); + my $metabib_metarecord_source_map_table = metabib::metarecord_source_map->table; + my $asset_call_number_table = asset::call_number->table; + + my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value',"$index_col"); my $fts_where = $fts->sql_where_clause(); my @fts_ranks = $fts->fts_rank; @@ -44,16 +55,16 @@ sub search_full_rec { } my $where = join(' OR ', @wheres); - my $select = "SELECT record, sum($rank) FROM $table WHERE $where GROUP BY 1 ORDER BY 2 DESC;"; + my $select = "SELECT record, sum($rank) FROM $search_table WHERE $where GROUP BY 1 ORDER BY 2 DESC;"; $log->debug("Search SQL :: [$select]",DEBUG); my $recs = metabib::full_rec->db_Main->selectall_arrayref($select, {}, @binds); $log->debug("Search yielded ".scalar(@$recs)." results.",DEBUG); - $client->respond( [ @$recs[0 .. $window - 1] ] ); + $client->respond_complete( [ @$recs[0 .. $window - 1] ] ); - OpenSRF::Utils::Cache->new->put_cache( $recs ); + OpenSRF::Utils::Cache->new->put_cache( $cache_key => $recs ); return undef; @@ -71,29 +82,42 @@ __PACKAGE__->register_method( stream => 1, ); + +# XXX factored most of the PG dependant stuff out of here... need to find a way to do "dependants". sub search_class_fts { my $self = shift; my $client = shift; - my $term = shift; - my $ou = shift; - my $ou_type = shift; - my $window = shift; - $window = 100 unless (defined $window); + my %args = @_; + my $term = $args{term}; + my $ou = $args{org_unit}; + my $ou_type = $args{depth}; + my $limit = $args{limit} || 100; + my $offset = $args{offset} || 0; + + + (my $search_class = $self->api_name) =~ s/.*metabib.(\w+).search_fts.*/$1/o; + my $cache_key = md5_hex($search_class.$term.$ou.$ou_type); - (my $class = $self->api_name) =~ s/.*metabib.(\w+).search_fts.*/$1/o; - my $cache_key = md5_hex($class.$term.$ou.$ou_type); + my $cached_recs = OpenSRF::Utils::Cache->new->get_cache( $cache_key ); + return [ @$cached_recs[$offset .. $limit - 1] ] if (defined $cached_recs); - $log->debug("Cache key for $class search of '$term' at ($ou,$ou_type) will be $cache_key", DEBUG); + $log->debug("Cache key for $search_class search of '$term' at ($ou,$ou_type) will be $cache_key", DEBUG); my $descendants = defined($ou_type) ? "actor.org_unit_descendants($ou, $ou_type)" : "actor.org_unit_descendants($ou)"; my $class = $self->{cdbi}; - my $table = $class->table; + my $search_table = $class->table; - my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value','index_vector'); + my $metabib_metarecord_source_map_table = metabib::metarecord_source_map->table; + my $asset_call_number_table = asset::call_number->table; + + my ($index_col) = $class->columns('FTS'); + $index_col ||= 'value'; + + my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value', "$index_col"); my $fts_where = $fts->sql_where_clause; my @fts_ranks = $fts->fts_rank; @@ -102,9 +126,9 @@ sub search_class_fts { my $select = <<" SQL"; SELECT m.metarecord, sum($rank)/count(m.source) - FROM $table f, - metabib.metarecord_source_map m, - asset.call_number cn, + FROM $search_table f, + $metabib_metarecord_source_map_table m, + $asset_call_number_table cn, $descendants d WHERE $fts_where AND m.source = f.source @@ -120,9 +144,9 @@ sub search_class_fts { $log->debug("Search yielded ".scalar(@$recs)." results.",DEBUG); - $client->respond( [ @$recs[0 .. $window - 1] ] ); + $client->respond_complete( [ @$recs[$offset .. $limit - 1] ] ); - OpenSRF::Utils::Cache->new->put_cache( $recs ); + OpenSRF::Utils::Cache->new->put_cache( $cache_key => $recs ); return undef; @@ -156,31 +180,48 @@ __PACKAGE__->register_method( cdbi => 'metabib::keyword_field_entry', ); +# XXX factored most of the PG dependant stuff out of here... need to find a way to do "dependants". sub search_class_fts_count { my $self = shift; my $client = shift; - my $term = shift; - my $ou = shift; - my $ou_type = shift; + my %args = @_; + + my $term = $args{term}; + my $ou = $args{org_unit}; + my $ou_type = $args{depth}; + my $limit = $args{limit} || 100; + my $offset = $args{offset} || 0; my $descendants = defined($ou_type) ? "actor.org_unit_descendants($ou, $ou_type)" : "actor.org_unit_descendants($ou)"; + (my $search_class = $self->api_name) =~ s/.*metabib.(\w+).search_fts.*/$1/o; + my $cache_key = md5_hex($search_class.$term.$ou.$ou_type.'_COUNT_'); + + my $cached_recs = OpenSRF::Utils::Cache->new->get_cache( $cache_key ); + return $cached_recs if (defined $cached_recs); + my $class = $self->{cdbi}; - my $table = $class->table; + my $search_table = $class->table; + + my $metabib_metarecord_source_map_table = metabib::metarecord_source_map->table; + my $asset_call_number_table = asset::call_number->table; - my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value','index_vector'); + my ($index_col) = $class->columns('FTS'); + $index_col ||= 'value'; + + my $fts = OpenILS::Application::Storage::FTS->compile($term, 'value',"$index_col"); my $fts_where = $fts->sql_where_clause; # XXX test an "EXISTS version of descendant checking... my $select = <<" SQL"; SELECT count(distinct m.metarecord) - FROM $table f, - metabib.metarecord_source_map m, - asset.call_number cn, + FROM $search_table f, + $metabib_metarecord_source_map_table m, + $asset_call_number_table cn, $descendants d WHERE $fts_where AND m.source = f.source @@ -194,6 +235,8 @@ sub search_class_fts_count { $log->debug("Count Search yielded $recs results.",DEBUG); + OpenSRF::Utils::Cache->new->put_cache( $cache_key => $recs ); + return $recs; } diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Storage/WORM.pm b/Open-ILS/src/perlmods/OpenILS/Application/Storage/WORM.pm index 43354720dd..4646bf572e 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Storage/WORM.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Storage/WORM.pm @@ -56,6 +56,7 @@ my @fp_mods_xpath = ( $text =~ s/\b(?:the|an?)\b//sgo; $text =~ s/\[.[^\]]+\]//sgo; $text =~ s/\s*[;\/\.]*$//sgo; + $text =~ s/\pM+//gso; }; ', }, @@ -69,7 +70,8 @@ my @fp_mods_xpath = ( $text = lc($text); $text =~ s/\s+/ /sgo; $text =~ s/^\s*(.+)\s*$/$1/sgo; - ($text) = split ",", $text; + $text =~ s/,?\s+.*$//sgo; + $text =~ s/\pM+//gso; }; ', }, @@ -95,6 +97,7 @@ my @fp_mods_xpath = ( $text =~ s/\b(?:the|an?)\b//sgo; $text =~ s/\[.[^\]]+\]//sgo; $text =~ s/\s*[;\/\.]*$//sgo; + $text =~ s/\pM+//gso; }; ', }, @@ -110,7 +113,8 @@ my @fp_mods_xpath = ( $text = lc($text); $text =~ s/\s+/ /sgo; $text =~ s/^\s*(.+)\s*$/$1/sgo; - ($text) = split ",", $text; + $text =~ s/,?\s+.*$//sgo; + $text =~ s/\pM+//gso; }; ', }, @@ -171,9 +175,13 @@ my $begin; my $commit; my $rollback; my $lookup; -my $fetch_entry; my $update_entry; +my $mr_lookup; +my $mr_create; +my $create_source_map; +my $sm_lookup; my $rm_old_rd; +my $rm_old_sm; my $rm_old_fr; my $rm_old_tr; my $rm_old_ar; @@ -209,12 +217,20 @@ sub wormize { unless ($commit); $rollback = $self->method_lookup( 'open-ils.storage.transaction.rollback') unless ($rollback); + $sm_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.search.source') + unless ($sm_lookup); + $mr_lookup = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.search.fingerprint') + unless ($mr_lookup); + $mr_create = $self->method_lookup('open-ils.storage.direct.metabib.metarecord.create') + unless ($mr_create); + $create_source_map = $self->method_lookup('open-ils.storage.direct.metabib.metarecord_source_map.batch.create') + unless ($create_source_map); $lookup = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.retrieve') unless ($lookup); - $fetch_entry = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.retrieve') - unless ($update_entry); $update_entry = $self->method_lookup('open-ils.storage.direct.biblio.record_entry.batch.update') unless ($update_entry); + $rm_old_sm = $self->method_lookup( 'open-ils.storage.direct.metabib.metarecord_source_map.mass_delete') + unless ($rm_old_sm); $rm_old_rd = $self->method_lookup( 'open-ils.storage.direct.metabib.record_descriptor.mass_delete') unless ($rm_old_rd); $rm_old_fr = $self->method_lookup( 'open-ils.storage.direct.metabib.full_rec.mass_delete') @@ -251,27 +267,46 @@ sub wormize { throw OpenSRF::EX::PANIC ("WoRM Couldn't BEGIN transaction!") }; + my @source_maps; my @entry_list; my @rd_list; my @ns_list; my @mods_data; my $ret = 0; - for my $marc ( $lookup->run(@docids) ) { + for my $entry ( $lookup->run(@docids) ) { # step -1: grab the doc from storage - next unless ($marc); + next unless ($entry); - my $xml = $marc->marc; - my $docid = $marc->id; + my $xml = $entry->marc; + my $docid = $entry->id; my $marcdoc = $parser->parse_string($xml); my $modsdoc = $mods_sheet->transform($marcdoc); my $mods = $modsdoc->documentElement; $mods->setNamespace( "http://www.loc.gov/mods/", "mods", 1 ); - my ($entry) = $fetch_entry->run($docid); $entry->fingerprint( fingerprint_mods( $mods ) ); push @entry_list, $entry; + my ($mr) = $mr_lookup->run( $entry->fingerprint ); + if (!@$mr) { + $mr = new Fieldmapper::metabib::metarecord; + $mr->fingerprint( $entry->fingerprint ); + $mr->master_record( $entry->id ); + my ($new_mr) = $mr_create->run($mr); + $mr->id($new_mr); + unless (defined $mr) { + throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord.create!") + } + } else { + $mr = $$mr[0]; + } + + my $sm = new Fieldmapper::metabib::metarecord_source_map; + $sm->metarecord( $mr->id ); + $sm->source( $entry->id ); + push @source_maps, $sm; + my $ldr = $marcdoc->documentElement->getChildrenByTagName('leader')->pop->textContent; my $oo8 = $marcdoc->documentElement->findvalue('//*[local-name()="controlfield" and @tag="008"]'); @@ -296,11 +331,17 @@ sub wormize { $rm_old_rd->run( { record => \@docids } ); $rm_old_fr->run( { record => \@docids } ); + $rm_old_sm->run( { source => \@docids } ); $rm_old_tr->run( { source => \@docids } ); $rm_old_ar->run( { source => \@docids } ); $rm_old_sr->run( { source => \@docids } ); $rm_old_kr->run( { source => \@docids } ); + my ($sm) = $create_source_map->run(@source_maps); + unless (defined $sm) { + throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.metabib.metarecord_source_map.batch.create!") + } + my ($re) = $update_entry->run(@entry_list); unless (defined $re) { throw OpenSRF::EX::PANIC ("Couldn't run open-ils.storage.direct.biblio.record_entry.batch.update!") diff --git a/Open-ILS/src/sql/Postgres/002.schema.config.sql b/Open-ILS/src/sql/Postgres/002.schema.config.sql index 35c7f80e2e..b462eb68d0 100644 --- a/Open-ILS/src/sql/Postgres/002.schema.config.sql +++ b/Open-ILS/src/sql/Postgres/002.schema.config.sql @@ -46,12 +46,12 @@ INSERT INTO config.identification_type ( name ) VALUES ( 'State ID' ); INSERT INTO config.identification_type ( name ) VALUES ( 'SSN' ); CREATE TABLE config.rule_circ_duration ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE CHECK ( name ~ '^\\w+$' ), - extended_duration INTERVAL NOT NULL, - normal_duration INTERVAL NOT NULL, - short_duration INTERVAL NOT NULL, - max_renewals INT NOT NULL + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE CHECK ( name ~ '^\\w+$' ), + extended INTERVAL NOT NULL, + normal INTERVAL NOT NULL, + short INTERVAL NOT NULL, + max_renewals INT NOT NULL ); CREATE TABLE config.rule_max_fine ( @@ -61,11 +61,11 @@ CREATE TABLE config.rule_max_fine ( ); CREATE TABLE config.rule_recuring_fine ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE CHECK ( name ~ '^\\w+$' ), - high_amount NUMERIC(6,2) NOT NULL, - normal_amount NUMERIC(6,2) NOT NULL, - low_amount NUMERIC(6,2) NOT NULL + id SERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE CHECK ( name ~ '^\\w+$' ), + high NUMERIC(6,2) NOT NULL, + normal NUMERIC(6,2) NOT NULL, + low NUMERIC(6,2) NOT NULL ); CREATE TABLE config.rule_age_hold_protect ( diff --git a/Open-ILS/src/sql/Postgres/005.schema.actors.sql b/Open-ILS/src/sql/Postgres/005.schema.actors.sql index 576d2265f9..3774db32a3 100644 --- a/Open-ILS/src/sql/Postgres/005.schema.actors.sql +++ b/Open-ILS/src/sql/Postgres/005.schema.actors.sql @@ -11,6 +11,8 @@ CREATE TABLE actor.usr ( usrname TEXT NOT NULL UNIQUE, email TEXT CHECK (email ~ $re$^[[:alnum:]_\.]+@[[:alnum:]_]+(?:\.[[:alnum:]_])+$$re$), passwd TEXT NOT NULL, + ident_type INT NOT NULL REFERENCES config.identifcation_type (id), + ident_value TEXT NOT NULL, prefix TEXT, first_given_name TEXT NOT NULL, second_given_name TEXT, @@ -58,8 +60,12 @@ CREATE TRIGGER actor_crypt_pw_insert_trigger EXECUTE PROCEDURE actor.crypt_pw_insert (); -- Just so that there is a user... -INSERT INTO actor.usr ( class, card, usrid, usrname, passwd, first_given_name, family_name, gender, dob, master_account, super_user ) - VALUES ( 3, 1,'admin', 'admin', 'open-ils', 'Administrator', '', 'm', '1979-01-22', TRUE, TRUE ); +INSERT INTO actor.usr ( class, card, usrid, usrname, passwd, first_given_name, family_name, gender, dob, master_account, super_user, ident_type, ident_value ) + VALUES ( 3, 1,'admin', 'admin', 'open-ils', 'Administrator', '', 'm', '1979-01-22', TRUE, TRUE, 1, 'identification' ); +INSERT INTO actor.usr ( class, card, usrid, usrname, passwd, first_given_name, family_name, gender, dob, master_account, super_user, ident_type, ident_value ) + VALUES ( 3, 1,'demo', 'demo', 'demo', 'demo', 'user', 'm', '1979-01-22', FALSE, TRUE, 1, 'identification' ); +INSERT INTO actor.usr ( class, card, usrid, usrname, passwd, first_given_name, family_name, gender, dob, master_account, super_user, ident_type, ident_value ) + VALUES ( 3, 1,'athens', 'athens', 'athens', 'athens', 'user', 'm', '1979-01-22', FALSE, TRUE, 1, 'identification' ); CREATE TABLE actor.usr_class ( id SERIAL PRIMARY KEY, -- 2.43.2