From cf3c03004915f7ab8c73c3dff90c8ddc7375186d Mon Sep 17 00:00:00 2001 From: dbs Date: Fri, 9 Jul 2010 15:42:54 +0000 Subject: [PATCH] New feature: maintain control numbers (001, 003, 035) for MARC records If a site wishes to maintain their control numbers and control number identifiers in authority, bibliographic, and MFHD MARC records according to the MARC21 specification, they will now be able to: 1. Enable the global flag 'cat.maintain_control_numbers' to turn on control number maintenance. This is disabled by default, so no changes will be made by the system until the flag is enabled. 2. (Optionally) Set the cat.marc_control_number_identifier OU setting to override the control number identifier for records owned by that OU; if this is not set, then the control number identifier defaults to the OU shortname. 3. Set the owner field for the records that are to be controlled; otherwise they will fall back to the global default which is currently hardcoded as 'EVRGRN'. git-svn-id: svn://svn.open-ils.org/ILS/trunk@16899 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/src/sql/Pg/002.functions.config.sql | 120 +++++++++++++ Open-ILS/src/sql/Pg/002.schema.config.sql | 2 +- Open-ILS/src/sql/Pg/010.schema.biblio.sql | 1 + Open-ILS/src/sql/Pg/011.schema.authority.sql | 4 +- Open-ILS/src/sql/Pg/210.schema.serials.sql | 4 +- Open-ILS/src/sql/Pg/950.data.seed-values.sql | 26 +++ .../0329.schema.maintain_control_numbers.sql | 158 ++++++++++++++++++ 7 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 Open-ILS/src/sql/Pg/upgrade/0329.schema.maintain_control_numbers.sql diff --git a/Open-ILS/src/sql/Pg/002.functions.config.sql b/Open-ILS/src/sql/Pg/002.functions.config.sql index e6a525cd23..5a375a859d 100644 --- a/Open-ILS/src/sql/Pg/002.functions.config.sql +++ b/Open-ILS/src/sql/Pg/002.functions.config.sql @@ -468,5 +468,125 @@ BEGIN END; $func$ LANGUAGE PLPGSQL; +CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$ +use strict; +use MARC::Record; +use MARC::File::XML; +use Encode; + +my $record = MARC::Record->new_from_xml($_TD->{new}{marc}); +my $schema = $_TD->{table_schema}; +my $rec_id = $_TD->{new}{id}; + +# Short-circuit if maintaining control numbers per MARC21 spec is not enabled +my $enable = spi_exec_query("SELECT enabled FROM config.global_flag WHERE name = 'cat.maintain_control_numbers'"); +if (!($enable->{processed}) or $enable->{rows}[0]->{enabled} eq 'f') { + return; +} + +# Get the control number identifier from an OU setting based on $_TD->{new}{owner} +my $ou_cni = 'EVRGRN'; + +# bre.owner can be null, so fall back to the consortial setting +my $owner = $_TD->{new}{owner} || 1; + +my $ous_rv = spi_exec_query("SELECT value FROM actor.org_unit_ancestor_setting('cat.marc_control_number_identifier', $owner)"); +if ($ous_rv->{processed}) { + $ou_cni = $ous_rv->{rows}[0]->{value}; + $ou_cni =~ s/"//g; # Stupid VIM syntax highlighting" +} else { + # Fall back to the shortname of the OU if there was no OU setting + $ous_rv = spi_exec_query("SELECT shortname FROM actor.org_unit WHERE id = $owner"); + if ($ous_rv->{processed}) { + $ou_cni = $ous_rv->{rows}[0]->{shortname}; + } +} + +my ($create, $munge) = (0, 0); +my ($orig_001, $orig_003) = ('', ''); + +# Incoming MARC records may have multiple 001s or 003s, despite the spec +my @control_ids = $record->field('003'); +my @scns = $record->field('035'); + +foreach my $id_field ('001', '003') { + my $spec_value; + my @controls = $record->field($id_field); + + if ($id_field eq '001') { + $spec_value = $rec_id; + } else { + $spec_value = $ou_cni; + } + + # Create the 001/003 if none exist + if (scalar(@controls) == 0) { + $record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value)); + $create = 1; + } elsif (scalar(@controls) > 1) { + # Do we already have the right 001/003 value in the existing set? + unless (grep $_->data() eq $spec_value, @controls) { + $munge = 1; + } + + # Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match + foreach my $control (@controls) { + unless ($control->data() eq $spec_value) { + $record->delete_field($control); + } + } + } else { + # Only one field; check to see if we need to munge it + unless (grep $_->data() eq $spec_value, @controls) { + $munge = 1; + } + } +} + +# Now, if we need to munge the 001, we will first push the existing 001/003 into the 035 +if ($munge) { + my $scn = "(" . $record->field('003')->data() . ")" . $record->field('001')->data(); + + # Do not create duplicate 035 fields + unless (grep $_->subfield('a') eq $scn, @scns) { + $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn)); + } +} + +# Set the 001/003 and update the MARC +if ($create or $munge) { + $record->field('001')->data($rec_id); + $record->field('003')->data($ou_cni); + + my $xml = $record->as_xml_record(); + $xml =~ s/\n//sgo; + $xml =~ s/^<\?xml.+\?\s*>//go; + $xml =~ s/>\s+entityize() + # to avoid having to set PERL5LIB for PostgreSQL as well + + # If we are going to convert non-ASCII characters to XML entities, + # we had better be dealing with a UTF8 string to begin with + $xml = decode_utf8($xml); + + $xml = NFC($xml); + + # Convert raw ampersands to entities + $xml =~ s/&(?!\S+;)/&/gso; + + # Convert Unicode characters to entities + $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe; + + $xml =~ s/[\x00-\x1f]//go; + $_TD->{new}{marc} = $xml; + + return "MODIFY"; +} + +return; +$func$ LANGUAGE PLPERLU; + COMMIT; diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql index aabe211310..7c54ad6ea3 100644 --- a/Open-ILS/src/sql/Pg/002.schema.config.sql +++ b/Open-ILS/src/sql/Pg/002.schema.config.sql @@ -68,7 +68,7 @@ CREATE TABLE config.upgrade_log ( install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() ); -INSERT INTO config.upgrade_log (version) VALUES ('0328'); -- phasefx +INSERT INTO config.upgrade_log (version) VALUES ('0329'); -- dbs CREATE TABLE config.bib_source ( id SERIAL PRIMARY KEY, diff --git a/Open-ILS/src/sql/Pg/010.schema.biblio.sql b/Open-ILS/src/sql/Pg/010.schema.biblio.sql index 2b1df69b15..428b575481 100644 --- a/Open-ILS/src/sql/Pg/010.schema.biblio.sql +++ b/Open-ILS/src/sql/Pg/010.schema.biblio.sql @@ -63,6 +63,7 @@ CREATE INDEX biblio_record_entry_fp_idx ON biblio.record_entry ( fingerprint ); CREATE UNIQUE INDEX biblio_record_unique_tcn ON biblio.record_entry (tcn_value) WHERE deleted = FALSE OR deleted IS FALSE; CREATE TRIGGER a_marcxml_is_well_formed BEFORE INSERT OR UPDATE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE biblio.check_marcxml_well_formed(); CREATE TRIGGER b_maintain_901 BEFORE INSERT OR UPDATE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_901(); +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); CREATE TABLE biblio.record_note ( id BIGSERIAL PRIMARY KEY, diff --git a/Open-ILS/src/sql/Pg/011.schema.authority.sql b/Open-ILS/src/sql/Pg/011.schema.authority.sql index b2a2e76489..4c99321c09 100644 --- a/Open-ILS/src/sql/Pg/011.schema.authority.sql +++ b/Open-ILS/src/sql/Pg/011.schema.authority.sql @@ -32,13 +32,15 @@ CREATE TABLE authority.record_entry ( deleted BOOL NOT NULL DEFAULT FALSE, source INT, marc TEXT NOT NULL, - last_xact_id TEXT NOT NULL + last_xact_id TEXT NOT NULL, + owner INT ); CREATE INDEX authority_record_entry_creator_idx ON authority.record_entry ( creator ); CREATE INDEX authority_record_entry_editor_idx ON authority.record_entry ( editor ); CREATE UNIQUE INDEX authority_record_unique_tcn ON authority.record_entry (arn_source,arn_value) WHERE deleted = FALSE OR deleted IS FALSE; CREATE TRIGGER a_marcxml_is_well_formed BEFORE INSERT OR UPDATE ON authority.record_entry FOR EACH ROW EXECUTE PROCEDURE biblio.check_marcxml_well_formed(); CREATE TRIGGER b_maintain_901 BEFORE INSERT OR UPDATE ON authority.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_901(); +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON authority.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); CREATE TABLE authority.bib_linking ( id BIGSERIAL PRIMARY KEY, diff --git a/Open-ILS/src/sql/Pg/210.schema.serials.sql b/Open-ILS/src/sql/Pg/210.schema.serials.sql index d3f3d6abf2..b3ed598063 100644 --- a/Open-ILS/src/sql/Pg/210.schema.serials.sql +++ b/Open-ILS/src/sql/Pg/210.schema.serials.sql @@ -18,12 +18,14 @@ CREATE TABLE serial.record_entry ( active BOOL NOT NULL DEFAULT TRUE, deleted BOOL NOT NULL DEFAULT FALSE, marc TEXT, - last_xact_id TEXT NOT NULL + last_xact_id TEXT NOT NULL, + owner INT ); CREATE INDEX serial_record_entry_creator_idx ON serial.record_entry ( creator ); CREATE INDEX serial_record_entry_editor_idx ON serial.record_entry ( editor ); CREATE INDEX serial_record_entry_owning_lib_idx ON serial.record_entry ( owning_lib, deleted ); CREATE TRIGGER b_maintain_901 BEFORE INSERT OR UPDATE ON serial.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_901(); +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON serial.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); CREATE RULE protect_mfhd_delete AS ON DELETE TO serial.record_entry DO INSTEAD UPDATE serial.record_entry SET deleted = true WHERE old.id = serial.record_entry.id; diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index d3d95056c7..615952452c 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -5764,6 +5764,17 @@ INSERT INTO config.global_flag (name,label,enabled) TRUE ); +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'cat.maintain_control_numbers', + oils_i18n_gettext( + 'cat.maintain_control_numbers', + 'Cat: Maintain 001/003/035 according to the MARC21 specification', + 'cgf', + 'label' + ) + ); + INSERT INTO config.usr_setting_type (name,opac_visible,label,description,datatype) VALUES ( 'history.circ.retention_age', @@ -5809,6 +5820,21 @@ VALUES ( 'interval' ); +INSERT INTO config.org_unit_setting_type ( name, label, description, datatype ) VALUES ( + 'cat.marc_control_number_identifier', + oils_i18n_gettext( + 'cat.marc_control_number_identifier', + 'Cat: Defines the control number identifier used in 003 and 035 fields.', + 'coust', + 'label'), + oils_i18n_gettext( + 'cat.marc_control_number_identifier', + 'Cat: Defines the control number identifier used in 003 and 035 fields.', + 'coust', + 'description'), + 'string' +); + -- 0311.data.query-seed-datatypes.sql -- Define the most common datatypes in query.datatype. Note that none of -- these stock datatypes specifies a width or precision. diff --git a/Open-ILS/src/sql/Pg/upgrade/0329.schema.maintain_control_numbers.sql b/Open-ILS/src/sql/Pg/upgrade/0329.schema.maintain_control_numbers.sql new file mode 100644 index 0000000000..96777e0634 --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/0329.schema.maintain_control_numbers.sql @@ -0,0 +1,158 @@ +BEGIN; + +INSERT INTO config.upgrade_log (version) VALUES ('0329'); -- dbs + +ALTER TABLE authority.record_entry ADD COLUMN owner INT; +ALTER TABLE serial.record_entry ADD COLUMN owner INT; + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'cat.maintain_control_numbers', + oils_i18n_gettext( + 'cat.maintain_control_numbers', + 'Cat: Maintain 001/003/035 according to the MARC21 specification', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.org_unit_setting_type ( name, label, description, datatype ) VALUES ( + 'cat.marc_control_number_identifier', + oils_i18n_gettext( + 'cat.marc_control_number_identifier', + 'Cat: Defines the control number identifier used in 003 and 035 fields.', + 'coust', + 'label'), + oils_i18n_gettext( + 'cat.marc_control_number_identifier', + 'Cat: Defines the control number identifier used in 003 and 035 fields.', + 'coust', + 'description'), + 'string' +); + +CREATE OR REPLACE FUNCTION maintain_control_numbers() RETURNS TRIGGER AS $func$ +use strict; +use MARC::Record; +use MARC::File::XML; +use Encode; + +my $record = MARC::Record->new_from_xml($_TD->{new}{marc}); +my $schema = $_TD->{table_schema}; +my $rec_id = $_TD->{new}{id}; + +# Short-circuit if maintaining control numbers per MARC21 spec is not enabled +my $enable = spi_exec_query("SELECT enabled FROM config.global_flag WHERE name = 'cat.maintain_control_numbers'"); +if (!($enable->{processed}) or $enable->{rows}[0]->{enabled} eq 'f') { + return; +} + +# Get the control number identifier from an OU setting based on $_TD->{new}{owner} +my $ou_cni = 'EVRGRN'; + +# bre.owner can be null, so fall back to the consortial setting +my $owner = $_TD->{new}{owner} || 1; + +my $ous_rv = spi_exec_query("SELECT value FROM actor.org_unit_ancestor_setting('cat.marc_control_number_identifier', $owner)"); +if ($ous_rv->{processed}) { + $ou_cni = $ous_rv->{rows}[0]->{value}; + $ou_cni =~ s/"//g; # Stupid VIM syntax highlighting" +} else { + # Fall back to the shortname of the OU if there was no OU setting + $ous_rv = spi_exec_query("SELECT shortname FROM actor.org_unit WHERE id = $owner"); + if ($ous_rv->{processed}) { + $ou_cni = $ous_rv->{rows}[0]->{shortname}; + } +} + +my ($create, $munge) = (0, 0); +my ($orig_001, $orig_003) = ('', ''); + +# Incoming MARC records may have multiple 001s or 003s, despite the spec +my @control_ids = $record->field('003'); +my @scns = $record->field('035'); + +foreach my $id_field ('001', '003') { + my $spec_value; + my @controls = $record->field($id_field); + + if ($id_field eq '001') { + $spec_value = $rec_id; + } else { + $spec_value = $ou_cni; + } + + # Create the 001/003 if none exist + if (scalar(@controls) == 0) { + $record->insert_fields_ordered(MARC::Field->new($id_field, $spec_value)); + $create = 1; + } elsif (scalar(@controls) > 1) { + # Do we already have the right 001/003 value in the existing set? + unless (grep $_->data() eq $spec_value, @controls) { + $munge = 1; + } + + # Delete the other fields, as with more than 1 001/003 we do not know which 003/001 to match + foreach my $control (@controls) { + unless ($control->data() eq $spec_value) { + $record->delete_field($control); + } + } + } else { + # Only one field; check to see if we need to munge it + unless (grep $_->data() eq $spec_value, @controls) { + $munge = 1; + } + } +} + +# Now, if we need to munge the 001, we will first push the existing 001/003 into the 035 +if ($munge) { + my $scn = "(" . $record->field('003')->data() . ")" . $record->field('001')->data(); + + # Do not create duplicate 035 fields + unless (grep $_->subfield('a') eq $scn, @scns) { + $record->insert_fields_ordered(MARC::Field->new('035', '', '', 'a' => $scn)); + } +} + +# Set the 001/003 and update the MARC +if ($create or $munge) { + $record->field('001')->data($rec_id); + $record->field('003')->data($ou_cni); + + my $xml = $record->as_xml_record(); + $xml =~ s/\n//sgo; + $xml =~ s/^<\?xml.+\?\s*>//go; + $xml =~ s/>\s+entityize() + # to avoid having to set PERL5LIB for PostgreSQL as well + + # If we are going to convert non-ASCII characters to XML entities, + # we had better be dealing with a UTF8 string to begin with + $xml = decode_utf8($xml); + + $xml = NFC($xml); + + # Convert raw ampersands to entities + $xml =~ s/&(?!\S+;)/&/gso; + + # Convert Unicode characters to entities + $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe; + + $xml =~ s/[\x00-\x1f]//go; + $_TD->{new}{marc} = $xml; + + return "MODIFY"; +} + +return; +$func$ LANGUAGE PLPERLU; + +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON authority.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); +CREATE TRIGGER c_maintain_control_numbers BEFORE INSERT OR UPDATE ON serial.record_entry FOR EACH ROW EXECUTE PROCEDURE maintain_control_numbers(); + +COMMIT; -- 2.43.2